drivers/net/ethernet/brocade/bna/bnad.c

   1 /*
   2  * Linux network driver for QLogic BR-series Converged Network Adapter.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License (GPL) Version 2 as
   6  * published by the Free Software Foundation
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 /*
  14  * Copyright (c) 2005-2014 Brocade Communications Systems, Inc.
  15  * Copyright (c) 2014-2015 QLogic Corporation
  16  * All rights reserved
  17  * www.qlogic.com
  18  */
  19 #include <linux/bitops.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/skbuff.h>
  22 #include <linux/etherdevice.h>
  23 #include <linux/in.h>
  24 #include <linux/ethtool.h>
  25 #include <linux/if_vlan.h>
  26 #include <linux/if_ether.h>
  27 #include <linux/ip.h>
  28 #include <linux/prefetch.h>
  29 #include <linux/module.h>
  30
  31 #include "bnad.h"
  32 #include "bna.h"
  33 #include "cna.h"
  34
  35 static DEFINE_MUTEX(bnad_fwimg_mutex);
  36
  37 /*
  38  * Module params
  39  */
  40 static uint bnad_msix_disable;
  41 module_param(bnad_msix_disable, uint, 0444);
  42 MODULE_PARM_DESC(bnad_msix_disable, "Disable MSIX mode");
  43
  44 static uint bnad_ioc_auto_recover = 1;
  45 module_param(bnad_ioc_auto_recover, uint, 0444);
  46 MODULE_PARM_DESC(bnad_ioc_auto_recover, "Enable / Disable auto recovery");
  47
  48 static uint bna_debugfs_enable = 1;
  49 module_param(bna_debugfs_enable, uint, S_IRUGO | S_IWUSR);
  50 MODULE_PARM_DESC(bna_debugfs_enable, "Enables debugfs feature, default=1,"
  51                  " Range[false:0|true:1]");
  52
  53 /*
  54  * Global variables
  55  */
  56 static u32 bnad_rxqs_per_cq = 2;
  57 static atomic_t bna_id;
  58 static struct mutex bnad_list_mutex;
  59 static const u8 bnad_bcast_addr[] __aligned(2) =
  60         { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
  61
  62 /*
  63  * Local MACROS
  64  */
  65 #define BNAD_GET_MBOX_IRQ(_bnad)                                \
  66         (((_bnad)->cfg_flags & BNAD_CF_MSIX) ?                  \
  67          ((_bnad)->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector) : \
  68          ((_bnad)->pcidev->irq))
  69
  70 #define BNAD_FILL_UNMAPQ_MEM_REQ(_res_info, _num, _size)        \
  71 do {                                                            \
  72         (_res_info)->res_type = BNA_RES_T_MEM;                  \
  73         (_res_info)->res_u.mem_info.mem_type = BNA_MEM_T_KVA;   \
  74         (_res_info)->res_u.mem_info.num = (_num);               \
  75         (_res_info)->res_u.mem_info.len = (_size);              \
  76 } while (0)
  77
  78 static void
  79 bnad_add_to_list(struct bnad *bnad)
  80 {
  81         mutex_lock(&bnad_list_mutex);
  82         mutex_unlock(&bnad_list_mutex);
  83 }
  84
  85 static void
  86 bnad_remove_from_list(struct bnad *bnad)
  87 {
  88         mutex_lock(&bnad_list_mutex);
  89         mutex_unlock(&bnad_list_mutex);
  90 }
  91
  92 /*
  93  * Reinitialize completions in CQ, once Rx is taken down
  94  */
  95 static void
  96 bnad_cq_cleanup(struct bnad *bnad, struct bna_ccb *ccb)
  97 {
  98         struct bna_cq_entry *cmpl;
  99         int i;
 100
 101         for (i = 0; i < ccb->q_depth; i++) {
 102                 cmpl = &((struct bna_cq_entry *)ccb->sw_q)[i];
 103                 cmpl->valid = 0;
 104         }
 105 }
 106
 107 /* Tx Datapath functions */
 108
 109
 110 /* Caller should ensure that the entry at unmap_q[index] is valid */
 111 static u32
 112 bnad_tx_buff_unmap(struct bnad *bnad,
 113                               struct bnad_tx_unmap *unmap_q,
 114                               u32 q_depth, u32 index)
 115 {
 116         struct bnad_tx_unmap *unmap;
 117         struct sk_buff *skb;
 118         int vector, nvecs;
 119
 120         unmap = &unmap_q[index];
 121         nvecs = unmap->nvecs;
 122
 123         skb = unmap->skb;
 124         unmap->skb = NULL;
 125         unmap->nvecs = 0;
 126         dma_unmap_single(&bnad->pcidev->dev,
 127                 dma_unmap_addr(&unmap->vectors[0], dma_addr),
 128                 skb_headlen(skb), DMA_TO_DEVICE);
 129         dma_unmap_addr_set(&unmap->vectors[0], dma_addr, 0);
 130         nvecs--;
 131
 132         vector = 0;
 133         while (nvecs) {
 134                 vector++;
 135                 if (vector == BFI_TX_MAX_VECTORS_PER_WI) {
 136                         vector = 0;
 137                         BNA_QE_INDX_INC(index, q_depth);
 138                         unmap = &unmap_q[index];
 139                 }
 140
 141                 dma_unmap_page(&bnad->pcidev->dev,
 142                         dma_unmap_addr(&unmap->vectors[vector], dma_addr),
 143                         dma_unmap_len(&unmap->vectors[vector], dma_len),
 144                         DMA_TO_DEVICE);
 145                 dma_unmap_addr_set(&unmap->vectors[vector], dma_addr, 0);
 146                 nvecs--;
 147         }
 148
 149         BNA_QE_INDX_INC(index, q_depth);
 150
 151         return index;
 152 }
 153
 154 /*
 155  * Frees all pending Tx Bufs
 156  * At this point no activity is expected on the Q,
 157  * so DMA unmap & freeing is fine.
 158  */
 159 static void
 160 bnad_txq_cleanup(struct bnad *bnad, struct bna_tcb *tcb)
 161 {
 162         struct bnad_tx_unmap *unmap_q = tcb->unmap_q;
 163         struct sk_buff *skb;
 164         int i;
 165
 166         for (i = 0; i < tcb->q_depth; i++) {
 167                 skb = unmap_q[i].skb;
 168                 if (!skb)
 169                         continue;
 170                 bnad_tx_buff_unmap(bnad, unmap_q, tcb->q_depth, i);
 171
 172                 dev_kfree_skb_any(skb);
 173         }
 174 }
 175
 176 /*
 177  * bnad_txcmpl_process : Frees the Tx bufs on Tx completion
 178  * Can be called in a) Interrupt context
 179  *                  b) Sending context
 180  */
 181 static u32
 182 bnad_txcmpl_process(struct bnad *bnad, struct bna_tcb *tcb)
 183 {
 184         u32 sent_packets = 0, sent_bytes = 0;
 185         u32 wis, unmap_wis, hw_cons, cons, q_depth;
 186         struct bnad_tx_unmap *unmap_q = tcb->unmap_q;
 187         struct bnad_tx_unmap *unmap;
 188         struct sk_buff *skb;
 189
 190         /* Just return if TX is stopped */
 191         if (!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))
 192                 return 0;
 193
 194         hw_cons = *(tcb->hw_consumer_index);
 195         cons = tcb->consumer_index;
 196         q_depth = tcb->q_depth;
 197
 198         wis = BNA_Q_INDEX_CHANGE(cons, hw_cons, q_depth);
 199         BUG_ON(!(wis <= BNA_QE_IN_USE_CNT(tcb, tcb->q_depth)));
 200
 201         while (wis) {
 202                 unmap = &unmap_q[cons];
 203
 204                 skb = unmap->skb;
 205
 206                 sent_packets++;
 207                 sent_bytes += skb->len;
 208
 209                 unmap_wis = BNA_TXQ_WI_NEEDED(unmap->nvecs);
 210                 wis -= unmap_wis;
 211
 212                 cons = bnad_tx_buff_unmap(bnad, unmap_q, q_depth, cons);
 213                 dev_kfree_skb_any(skb);
 214         }
 215
 216         /* Update consumer pointers. */
 217         tcb->consumer_index = hw_cons;
 218
 219         tcb->txq->tx_packets += sent_packets;
 220         tcb->txq->tx_bytes += sent_bytes;
 221
 222         return sent_packets;
 223 }
 224
 225 static u32
 226 bnad_tx_complete(struct bnad *bnad, struct bna_tcb *tcb)
 227 {
 228         struct net_device *netdev = bnad->netdev;
 229         u32 sent = 0;
 230
 231         if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags))
 232                 return 0;
 233
 234         sent = bnad_txcmpl_process(bnad, tcb);
 235         if (sent) {
 236                 if (netif_queue_stopped(netdev) &&
 237                     netif_carrier_ok(netdev) &&
 238                     BNA_QE_FREE_CNT(tcb, tcb->q_depth) >=
 239                                     BNAD_NETIF_WAKE_THRESHOLD) {
 240                         if (test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) {
 241                                 netif_wake_queue(netdev);
 242                                 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup);
 243                         }
 244                 }
 245         }
 246
 247         if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)))
 248                 bna_ib_ack(tcb->i_dbell, sent);
 249
 250         smp_mb__before_atomic();
 251         clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags);
 252
 253         return sent;
 254 }
 255
 256 /* MSIX Tx Completion Handler */
 257 static irqreturn_t
 258 bnad_msix_tx(int irq, void *data)
 259 {
 260         struct bna_tcb *tcb = (struct bna_tcb *)data;
 261         struct bnad *bnad = tcb->bnad;
 262
 263         bnad_tx_complete(bnad, tcb);
 264
 265         return IRQ_HANDLED;
 266 }
 267
 268 static inline void
 269 bnad_rxq_alloc_uninit(struct bnad *bnad, struct bna_rcb *rcb)
 270 {
 271         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 272
 273         unmap_q->reuse_pi = -1;
 274         unmap_q->alloc_order = -1;
 275         unmap_q->map_size = 0;
 276         unmap_q->type = BNAD_RXBUF_NONE;
 277 }
 278
 279 /* Default is page-based allocation. Multi-buffer support - TBD */
 280 static int
 281 bnad_rxq_alloc_init(struct bnad *bnad, struct bna_rcb *rcb)
 282 {
 283         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 284         int order;
 285
 286         bnad_rxq_alloc_uninit(bnad, rcb);
 287
 288         order = get_order(rcb->rxq->buffer_size);
 289
 290         unmap_q->type = BNAD_RXBUF_PAGE;
 291
 292         if (bna_is_small_rxq(rcb->id)) {
 293                 unmap_q->alloc_order = 0;
 294                 unmap_q->map_size = rcb->rxq->buffer_size;
 295         } else {
 296                 if (rcb->rxq->multi_buffer) {
 297                         unmap_q->alloc_order = 0;
 298                         unmap_q->map_size = rcb->rxq->buffer_size;
 299                         unmap_q->type = BNAD_RXBUF_MULTI_BUFF;
 300                 } else {
 301                         unmap_q->alloc_order = order;
 302                         unmap_q->map_size =
 303                                 (rcb->rxq->buffer_size > 2048) ?
 304                                 PAGE_SIZE << order : 2048;
 305                 }
 306         }
 307
 308         BUG_ON((PAGE_SIZE << order) % unmap_q->map_size);
 309
 310         return 0;
 311 }
 312
 313 static inline void
 314 bnad_rxq_cleanup_page(struct bnad *bnad, struct bnad_rx_unmap *unmap)
 315 {
 316         if (!unmap->page)
 317                 return;
 318
 319         dma_unmap_page(&bnad->pcidev->dev,
 320                         dma_unmap_addr(&unmap->vector, dma_addr),
 321                         unmap->vector.len, DMA_FROM_DEVICE);
 322         put_page(unmap->page);
 323         unmap->page = NULL;
 324         dma_unmap_addr_set(&unmap->vector, dma_addr, 0);
 325         unmap->vector.len = 0;
 326 }
 327
 328 static inline void
 329 bnad_rxq_cleanup_skb(struct bnad *bnad, struct bnad_rx_unmap *unmap)
 330 {
 331         if (!unmap->skb)
 332                 return;
 333
 334         dma_unmap_single(&bnad->pcidev->dev,
 335                         dma_unmap_addr(&unmap->vector, dma_addr),
 336                         unmap->vector.len, DMA_FROM_DEVICE);
 337         dev_kfree_skb_any(unmap->skb);
 338         unmap->skb = NULL;
 339         dma_unmap_addr_set(&unmap->vector, dma_addr, 0);
 340         unmap->vector.len = 0;
 341 }
 342
 343 static void
 344 bnad_rxq_cleanup(struct bnad *bnad, struct bna_rcb *rcb)
 345 {
 346         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 347         int i;
 348
 349         for (i = 0; i < rcb->q_depth; i++) {
 350                 struct bnad_rx_unmap *unmap = &unmap_q->unmap[i];
 351
 352                 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 353                         bnad_rxq_cleanup_skb(bnad, unmap);
 354                 else
 355                         bnad_rxq_cleanup_page(bnad, unmap);
 356         }
 357         bnad_rxq_alloc_uninit(bnad, rcb);
 358 }
 359
 360 static u32
 361 bnad_rxq_refill_page(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc)
 362 {
 363         u32 alloced, prod, q_depth;
 364         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 365         struct bnad_rx_unmap *unmap, *prev;
 366         struct bna_rxq_entry *rxent;
 367         struct page *page;
 368         u32 page_offset, alloc_size;
 369         dma_addr_t dma_addr;
 370
 371         prod = rcb->producer_index;
 372         q_depth = rcb->q_depth;
 373
 374         alloc_size = PAGE_SIZE << unmap_q->alloc_order;
 375         alloced = 0;
 376
 377         while (nalloc--) {
 378                 unmap = &unmap_q->unmap[prod];
 379
 380                 if (unmap_q->reuse_pi < 0) {
 381                         page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
 382                                         unmap_q->alloc_order);
 383                         page_offset = 0;
 384                 } else {
 385                         prev = &unmap_q->unmap[unmap_q->reuse_pi];
 386                         page = prev->page;
 387                         page_offset = prev->page_offset + unmap_q->map_size;
 388                         get_page(page);
 389                 }
 390
 391                 if (unlikely(!page)) {
 392                         BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed);
 393                         rcb->rxq->rxbuf_alloc_failed++;
 394                         goto finishing;
 395                 }
 396
 397                 dma_addr = dma_map_page(&bnad->pcidev->dev, page, page_offset,
 398                                         unmap_q->map_size, DMA_FROM_DEVICE);
 399                 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
 400                         put_page(page);
 401                         BNAD_UPDATE_CTR(bnad, rxbuf_map_failed);
 402                         rcb->rxq->rxbuf_map_failed++;
 403                         goto finishing;
 404                 }
 405
 406                 unmap->page = page;
 407                 unmap->page_offset = page_offset;
 408                 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr);
 409                 unmap->vector.len = unmap_q->map_size;
 410                 page_offset += unmap_q->map_size;
 411
 412                 if (page_offset < alloc_size)
 413                         unmap_q->reuse_pi = prod;
 414                 else
 415                         unmap_q->reuse_pi = -1;
 416
 417                 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod];
 418                 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr);
 419                 BNA_QE_INDX_INC(prod, q_depth);
 420                 alloced++;
 421         }
 422
 423 finishing:
 424         if (likely(alloced)) {
 425                 rcb->producer_index = prod;
 426                 smp_mb();
 427                 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags)))
 428                         bna_rxq_prod_indx_doorbell(rcb);
 429         }
 430
 431         return alloced;
 432 }
 433
 434 static u32
 435 bnad_rxq_refill_skb(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc)
 436 {
 437         u32 alloced, prod, q_depth, buff_sz;
 438         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 439         struct bnad_rx_unmap *unmap;
 440         struct bna_rxq_entry *rxent;
 441         struct sk_buff *skb;
 442         dma_addr_t dma_addr;
 443
 444         buff_sz = rcb->rxq->buffer_size;
 445         prod = rcb->producer_index;
 446         q_depth = rcb->q_depth;
 447
 448         alloced = 0;
 449         while (nalloc--) {
 450                 unmap = &unmap_q->unmap[prod];
 451
 452                 skb = netdev_alloc_skb_ip_align(bnad->netdev, buff_sz);
 453
 454                 if (unlikely(!skb)) {
 455                         BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed);
 456                         rcb->rxq->rxbuf_alloc_failed++;
 457                         goto finishing;
 458                 }
 459
 460                 dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data,
 461                                           buff_sz, DMA_FROM_DEVICE);
 462                 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
 463                         dev_kfree_skb_any(skb);
 464                         BNAD_UPDATE_CTR(bnad, rxbuf_map_failed);
 465                         rcb->rxq->rxbuf_map_failed++;
 466                         goto finishing;
 467                 }
 468
 469                 unmap->skb = skb;
 470                 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr);
 471                 unmap->vector.len = buff_sz;
 472
 473                 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod];
 474                 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr);
 475                 BNA_QE_INDX_INC(prod, q_depth);
 476                 alloced++;
 477         }
 478
 479 finishing:
 480         if (likely(alloced)) {
 481                 rcb->producer_index = prod;
 482                 smp_mb();
 483                 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags)))
 484                         bna_rxq_prod_indx_doorbell(rcb);
 485         }
 486
 487         return alloced;
 488 }
 489
 490 static inline void
 491 bnad_rxq_post(struct bnad *bnad, struct bna_rcb *rcb)
 492 {
 493         struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q;
 494         u32 to_alloc;
 495
 496         to_alloc = BNA_QE_FREE_CNT(rcb, rcb->q_depth);
 497         if (!(to_alloc >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT))
 498                 return;
 499
 500         if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 501                 bnad_rxq_refill_skb(bnad, rcb, to_alloc);
 502         else
 503                 bnad_rxq_refill_page(bnad, rcb, to_alloc);
 504 }
 505
 506 #define flags_cksum_prot_mask (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \
 507                                         BNA_CQ_EF_IPV6 | \
 508                                         BNA_CQ_EF_TCP | BNA_CQ_EF_UDP | \
 509                                         BNA_CQ_EF_L4_CKSUM_OK)
 510
 511 #define flags_tcp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \
 512                                 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK)
 513 #define flags_tcp6 (BNA_CQ_EF_IPV6 | \
 514                                 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK)
 515 #define flags_udp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \
 516                                 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK)
 517 #define flags_udp6 (BNA_CQ_EF_IPV6 | \
 518                                 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK)
 519
 520 static void
 521 bnad_cq_drop_packet(struct bnad *bnad, struct bna_rcb *rcb,
 522                     u32 sop_ci, u32 nvecs)
 523 {
 524         struct bnad_rx_unmap_q *unmap_q;
 525         struct bnad_rx_unmap *unmap;
 526         u32 ci, vec;
 527
 528         unmap_q = rcb->unmap_q;
 529         for (vec = 0, ci = sop_ci; vec < nvecs; vec++) {
 530                 unmap = &unmap_q->unmap[ci];
 531                 BNA_QE_INDX_INC(ci, rcb->q_depth);
 532
 533                 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 534                         bnad_rxq_cleanup_skb(bnad, unmap);
 535                 else
 536                         bnad_rxq_cleanup_page(bnad, unmap);
 537         }
 538 }
 539
 540 static void
 541 bnad_cq_setup_skb_frags(struct bna_ccb *ccb, struct sk_buff *skb, u32 nvecs)
 542 {
 543         struct bna_rcb *rcb;
 544         struct bnad *bnad;
 545         struct bnad_rx_unmap_q *unmap_q;
 546         struct bna_cq_entry *cq, *cmpl;
 547         u32 ci, pi, totlen = 0;
 548
 549         cq = ccb->sw_q;
 550         pi = ccb->producer_index;
 551         cmpl = &cq[pi];
 552
 553         rcb = bna_is_small_rxq(cmpl->rxq_id) ? ccb->rcb[1] : ccb->rcb[0];
 554         unmap_q = rcb->unmap_q;
 555         bnad = rcb->bnad;
 556         ci = rcb->consumer_index;
 557
 558         /* prefetch header */
 559         prefetch(page_address(unmap_q->unmap[ci].page) +
 560                  unmap_q->unmap[ci].page_offset);
 561
 562         while (nvecs--) {
 563                 struct bnad_rx_unmap *unmap;
 564                 u32 len;
 565
 566                 unmap = &unmap_q->unmap[ci];
 567                 BNA_QE_INDX_INC(ci, rcb->q_depth);
 568
 569                 dma_unmap_page(&bnad->pcidev->dev,
 570                                dma_unmap_addr(&unmap->vector, dma_addr),
 571                                unmap->vector.len, DMA_FROM_DEVICE);
 572
 573                 len = ntohs(cmpl->length);
 574                 skb->truesize += unmap->vector.len;
 575                 totlen += len;
 576
 577                 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 578                                    unmap->page, unmap->page_offset, len);
 579
 580                 unmap->page = NULL;
 581                 unmap->vector.len = 0;
 582
 583                 BNA_QE_INDX_INC(pi, ccb->q_depth);
 584                 cmpl = &cq[pi];
 585         }
 586
 587         skb->len += totlen;
 588         skb->data_len += totlen;
 589 }
 590
 591 static inline void
 592 bnad_cq_setup_skb(struct bnad *bnad, struct sk_buff *skb,
 593                   struct bnad_rx_unmap *unmap, u32 len)
 594 {
 595         prefetch(skb->data);
 596
 597         dma_unmap_single(&bnad->pcidev->dev,
 598                         dma_unmap_addr(&unmap->vector, dma_addr),
 599                         unmap->vector.len, DMA_FROM_DEVICE);
 600
 601         skb_put(skb, len);
 602         skb->protocol = eth_type_trans(skb, bnad->netdev);
 603
 604         unmap->skb = NULL;
 605         unmap->vector.len = 0;
 606 }
 607
 608 static u32
 609 bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget)
 610 {
 611         struct bna_cq_entry *cq, *cmpl, *next_cmpl;
 612         struct bna_rcb *rcb = NULL;
 613         struct bnad_rx_unmap_q *unmap_q;
 614         struct bnad_rx_unmap *unmap = NULL;
 615         struct sk_buff *skb = NULL;
 616         struct bna_pkt_rate *pkt_rt = &ccb->pkt_rate;
 617         struct bnad_rx_ctrl *rx_ctrl = ccb->ctrl;
 618         u32 packets = 0, len = 0, totlen = 0;
 619         u32 pi, vec, sop_ci = 0, nvecs = 0;
 620         u32 flags, masked_flags;
 621
 622         prefetch(bnad->netdev);
 623
 624         cq = ccb->sw_q;
 625
 626         while (packets < budget) {
 627                 cmpl = &cq[ccb->producer_index];
 628                 if (!cmpl->valid)
 629                         break;
 630                 /* The 'valid' field is set by the adapter, only after writing
 631                  * the other fields of completion entry. Hence, do not load
 632                  * other fields of completion entry *before* the 'valid' is
 633                  * loaded. Adding the rmb() here prevents the compiler and/or
 634                  * CPU from reordering the reads which would potentially result
 635                  * in reading stale values in completion entry.
 636                  */
 637                 rmb();
 638
 639                 BNA_UPDATE_PKT_CNT(pkt_rt, ntohs(cmpl->length));
 640
 641                 if (bna_is_small_rxq(cmpl->rxq_id))
 642                         rcb = ccb->rcb[1];
 643                 else
 644                         rcb = ccb->rcb[0];
 645
 646                 unmap_q = rcb->unmap_q;
 647
 648                 /* start of packet ci */
 649                 sop_ci = rcb->consumer_index;
 650
 651                 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) {
 652                         unmap = &unmap_q->unmap[sop_ci];
 653                         skb = unmap->skb;
 654                 } else {
 655                         skb = napi_get_frags(&rx_ctrl->napi);
 656                         if (unlikely(!skb))
 657                                 break;
 658                 }
 659                 prefetch(skb);
 660
 661                 flags = ntohl(cmpl->flags);
 662                 len = ntohs(cmpl->length);
 663                 totlen = len;
 664                 nvecs = 1;
 665
 666                 /* Check all the completions for this frame.
 667                  * busy-wait doesn't help much, break here.
 668                  */
 669                 if (BNAD_RXBUF_IS_MULTI_BUFF(unmap_q->type) &&
 670                     (flags & BNA_CQ_EF_EOP) == 0) {
 671                         pi = ccb->producer_index;
 672                         do {
 673                                 BNA_QE_INDX_INC(pi, ccb->q_depth);
 674                                 next_cmpl = &cq[pi];
 675
 676                                 if (!next_cmpl->valid)
 677                                         break;
 678                                 /* The 'valid' field is set by the adapter, only
 679                                  * after writing the other fields of completion
 680                                  * entry. Hence, do not load other fields of
 681                                  * completion entry *before* the 'valid' is
 682                                  * loaded. Adding the rmb() here prevents the
 683                                  * compiler and/or CPU from reordering the reads
 684                                  * which would potentially result in reading
 685                                  * stale values in completion entry.
 686                                  */
 687                                 rmb();
 688
 689                                 len = ntohs(next_cmpl->length);
 690                                 flags = ntohl(next_cmpl->flags);
 691
 692                                 nvecs++;
 693                                 totlen += len;
 694                         } while ((flags & BNA_CQ_EF_EOP) == 0);
 695
 696                         if (!next_cmpl->valid)
 697                                 break;
 698                 }
 699                 packets++;
 700
 701                 /* TODO: BNA_CQ_EF_LOCAL ? */
 702                 if (unlikely(flags & (BNA_CQ_EF_MAC_ERROR |
 703                                                 BNA_CQ_EF_FCS_ERROR |
 704                                                 BNA_CQ_EF_TOO_LONG))) {
 705                         bnad_cq_drop_packet(bnad, rcb, sop_ci, nvecs);
 706                         rcb->rxq->rx_packets_with_error++;
 707
 708                         goto next;
 709                 }
 710
 711                 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 712                         bnad_cq_setup_skb(bnad, skb, unmap, len);
 713                 else
 714                         bnad_cq_setup_skb_frags(ccb, skb, nvecs);
 715
 716                 rcb->rxq->rx_packets++;
 717                 rcb->rxq->rx_bytes += totlen;
 718                 ccb->bytes_per_intr += totlen;
 719
 720                 masked_flags = flags & flags_cksum_prot_mask;
 721
 722                 if (likely
 723                     ((bnad->netdev->features & NETIF_F_RXCSUM) &&
 724                      ((masked_flags == flags_tcp4) ||
 725                       (masked_flags == flags_udp4) ||
 726                       (masked_flags == flags_tcp6) ||
 727                       (masked_flags == flags_udp6))))
 728                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 729                 else
 730                         skb_checksum_none_assert(skb);
 731
 732                 if ((flags & BNA_CQ_EF_VLAN) &&
 733                     (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
 734                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cmpl->vlan_tag));
 735
 736                 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type))
 737                         netif_receive_skb(skb);
 738                 else
 739                         napi_gro_frags(&rx_ctrl->napi);
 740
 741 next:
 742                 BNA_QE_INDX_ADD(rcb->consumer_index, nvecs, rcb->q_depth);
 743                 for (vec = 0; vec < nvecs; vec++) {
 744                         cmpl = &cq[ccb->producer_index];
 745                         cmpl->valid = 0;
 746                         BNA_QE_INDX_INC(ccb->producer_index, ccb->q_depth);
 747                 }
 748         }
 749
 750         napi_gro_flush(&rx_ctrl->napi, false);
 751         if (likely(test_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags)))
 752                 bna_ib_ack_disable_irq(ccb->i_dbell, packets);
 753
 754         bnad_rxq_post(bnad, ccb->rcb[0]);
 755         if (ccb->rcb[1])
 756                 bnad_rxq_post(bnad, ccb->rcb[1]);
 757
 758         return packets;
 759 }
 760
 761 static void
 762 bnad_netif_rx_schedule_poll(struct bnad *bnad, struct bna_ccb *ccb)
 763 {
 764         struct bnad_rx_ctrl *rx_ctrl = (struct bnad_rx_ctrl *)(ccb->ctrl);
 765         struct napi_struct *napi = &rx_ctrl->napi;
 766
 767         if (likely(napi_schedule_prep(napi))) {
 768                 __napi_schedule(napi);
 769                 rx_ctrl->rx_schedule++;
 770         }
 771 }
 772
 773 /* MSIX Rx Path Handler */
 774 static irqreturn_t
 775 bnad_msix_rx(int irq, void *data)
 776 {
 777         struct bna_ccb *ccb = (struct bna_ccb *)data;
 778
 779         if (ccb) {
 780                 ((struct bnad_rx_ctrl *)ccb->ctrl)->rx_intr_ctr++;
 781                 bnad_netif_rx_schedule_poll(ccb->bnad, ccb);
 782         }
 783
 784         return IRQ_HANDLED;
 785 }
 786
 787 /* Interrupt handlers */
 788
 789 /* Mbox Interrupt Handlers */
 790 static irqreturn_t
 791 bnad_msix_mbox_handler(int irq, void *data)
 792 {
 793         u32 intr_status;
 794         unsigned long flags;
 795         struct bnad *bnad = (struct bnad *)data;
 796
 797         spin_lock_irqsave(&bnad->bna_lock, flags);
 798         if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) {
 799                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
 800                 return IRQ_HANDLED;
 801         }
 802
 803         bna_intr_status_get(&bnad->bna, intr_status);
 804
 805         if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status))
 806                 bna_mbox_handler(&bnad->bna, intr_status);
 807
 808         spin_unlock_irqrestore(&bnad->bna_lock, flags);
 809
 810         return IRQ_HANDLED;
 811 }
 812
 813 static irqreturn_t
 814 bnad_isr(int irq, void *data)
 815 {
 816         int i, j;
 817         u32 intr_status;
 818         unsigned long flags;
 819         struct bnad *bnad = (struct bnad *)data;
 820         struct bnad_rx_info *rx_info;
 821         struct bnad_rx_ctrl *rx_ctrl;
 822         struct bna_tcb *tcb = NULL;
 823
 824         spin_lock_irqsave(&bnad->bna_lock, flags);
 825         if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) {
 826                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
 827                 return IRQ_NONE;
 828         }
 829
 830         bna_intr_status_get(&bnad->bna, intr_status);
 831
 832         if (unlikely(!intr_status)) {
 833                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
 834                 return IRQ_NONE;
 835         }
 836
 837         if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status))
 838                 bna_mbox_handler(&bnad->bna, intr_status);
 839
 840         spin_unlock_irqrestore(&bnad->bna_lock, flags);
 841
 842         if (!BNA_IS_INTX_DATA_INTR(intr_status))
 843                 return IRQ_HANDLED;
 844
 845         /* Process data interrupts */
 846         /* Tx processing */
 847         for (i = 0; i < bnad->num_tx; i++) {
 848                 for (j = 0; j < bnad->num_txq_per_tx; j++) {
 849                         tcb = bnad->tx_info[i].tcb[j];
 850                         if (tcb && test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))
 851                                 bnad_tx_complete(bnad, bnad->tx_info[i].tcb[j]);
 852                 }
 853         }
 854         /* Rx processing */
 855         for (i = 0; i < bnad->num_rx; i++) {
 856                 rx_info = &bnad->rx_info[i];
 857                 if (!rx_info->rx)
 858                         continue;
 859                 for (j = 0; j < bnad->num_rxp_per_rx; j++) {
 860                         rx_ctrl = &rx_info->rx_ctrl[j];
 861                         if (rx_ctrl->ccb)
 862                                 bnad_netif_rx_schedule_poll(bnad,
 863                                                             rx_ctrl->ccb);
 864                 }
 865         }
 866         return IRQ_HANDLED;
 867 }
 868
 869 /*
 870  * Called in interrupt / callback context
 871  * with bna_lock held, so cfg_flags access is OK
 872  */
 873 static void
 874 bnad_enable_mbox_irq(struct bnad *bnad)
 875 {
 876         clear_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags);
 877
 878         BNAD_UPDATE_CTR(bnad, mbox_intr_enabled);
 879 }
 880
 881 /*
 882  * Called with bnad->bna_lock held b'cos of
 883  * bnad->cfg_flags access.
 884  */
 885 static void
 886 bnad_disable_mbox_irq(struct bnad *bnad)
 887 {
 888         set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags);
 889
 890         BNAD_UPDATE_CTR(bnad, mbox_intr_disabled);
 891 }
 892
 893 static void
 894 bnad_set_netdev_perm_addr(struct bnad *bnad)
 895 {
 896         struct net_device *netdev = bnad->netdev;
 897
 898         ether_addr_copy(netdev->perm_addr, bnad->perm_addr);
 899         if (is_zero_ether_addr(netdev->dev_addr))
 900                 ether_addr_copy(netdev->dev_addr, bnad->perm_addr);
 901 }
 902
 903 /* Control Path Handlers */
 904
 905 /* Callbacks */
 906 void
 907 bnad_cb_mbox_intr_enable(struct bnad *bnad)
 908 {
 909         bnad_enable_mbox_irq(bnad);
 910 }
 911
 912 void
 913 bnad_cb_mbox_intr_disable(struct bnad *bnad)
 914 {
 915         bnad_disable_mbox_irq(bnad);
 916 }
 917
 918 void
 919 bnad_cb_ioceth_ready(struct bnad *bnad)
 920 {
 921         bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS;
 922         complete(&bnad->bnad_completions.ioc_comp);
 923 }
 924
 925 void
 926 bnad_cb_ioceth_failed(struct bnad *bnad)
 927 {
 928         bnad->bnad_completions.ioc_comp_status = BNA_CB_FAIL;
 929         complete(&bnad->bnad_completions.ioc_comp);
 930 }
 931
 932 void
 933 bnad_cb_ioceth_disabled(struct bnad *bnad)
 934 {
 935         bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS;
 936         complete(&bnad->bnad_completions.ioc_comp);
 937 }
 938
 939 static void
 940 bnad_cb_enet_disabled(void *arg)
 941 {
 942         struct bnad *bnad = (struct bnad *)arg;
 943
 944         netif_carrier_off(bnad->netdev);
 945         complete(&bnad->bnad_completions.enet_comp);
 946 }
 947
 948 void
 949 bnad_cb_ethport_link_status(struct bnad *bnad,
 950                         enum bna_link_status link_status)
 951 {
 952         bool link_up = false;
 953
 954         link_up = (link_status == BNA_LINK_UP) || (link_status == BNA_CEE_UP);
 955
 956         if (link_status == BNA_CEE_UP) {
 957                 if (!test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags))
 958                         BNAD_UPDATE_CTR(bnad, cee_toggle);
 959                 set_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags);
 960         } else {
 961                 if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags))
 962                         BNAD_UPDATE_CTR(bnad, cee_toggle);
 963                 clear_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags);
 964         }
 965
 966         if (link_up) {
 967                 if (!netif_carrier_ok(bnad->netdev)) {
 968                         uint tx_id, tcb_id;
 969                         netdev_info(bnad->netdev, "link up\n");
 970                         netif_carrier_on(bnad->netdev);
 971                         BNAD_UPDATE_CTR(bnad, link_toggle);
 972                         for (tx_id = 0; tx_id < bnad->num_tx; tx_id++) {
 973                                 for (tcb_id = 0; tcb_id < bnad->num_txq_per_tx;
 974                                       tcb_id++) {
 975                                         struct bna_tcb *tcb =
 976                                         bnad->tx_info[tx_id].tcb[tcb_id];
 977                                         u32 txq_id;
 978                                         if (!tcb)
 979                                                 continue;
 980
 981                                         txq_id = tcb->id;
 982
 983                                         if (test_bit(BNAD_TXQ_TX_STARTED,
 984                                                      &tcb->flags)) {
 985                                                 /*
 986                                                  * Force an immediate
 987                                                  * Transmit Schedule */
 988                                                 netif_wake_subqueue(
 989                                                                 bnad->netdev,
 990                                                                 txq_id);
 991                                                 BNAD_UPDATE_CTR(bnad,
 992                                                         netif_queue_wakeup);
 993                                         } else {
 994                                                 netif_stop_subqueue(
 995                                                                 bnad->netdev,
 996                                                                 txq_id);
 997                                                 BNAD_UPDATE_CTR(bnad,
 998                                                         netif_queue_stop);
 999                                         }
1000                                 }
1001                         }
1002                 }
1003         } else {
1004                 if (netif_carrier_ok(bnad->netdev)) {
1005                         netdev_info(bnad->netdev, "link down\n");
1006                         netif_carrier_off(bnad->netdev);
1007                         BNAD_UPDATE_CTR(bnad, link_toggle);
1008                 }
1009         }
1010 }
1011
1012 static void
1013 bnad_cb_tx_disabled(void *arg, struct bna_tx *tx)
1014 {
1015         struct bnad *bnad = (struct bnad *)arg;
1016
1017         complete(&bnad->bnad_completions.tx_comp);
1018 }
1019
1020 static void
1021 bnad_cb_tcb_setup(struct bnad *bnad, struct bna_tcb *tcb)
1022 {
1023         struct bnad_tx_info *tx_info =
1024                         (struct bnad_tx_info *)tcb->txq->tx->priv;
1025
1026         tcb->priv = tcb;
1027         tx_info->tcb[tcb->id] = tcb;
1028 }
1029
1030 static void
1031 bnad_cb_tcb_destroy(struct bnad *bnad, struct bna_tcb *tcb)
1032 {
1033         struct bnad_tx_info *tx_info =
1034                         (struct bnad_tx_info *)tcb->txq->tx->priv;
1035
1036         tx_info->tcb[tcb->id] = NULL;
1037         tcb->priv = NULL;
1038 }
1039
1040 static void
1041 bnad_cb_ccb_setup(struct bnad *bnad, struct bna_ccb *ccb)
1042 {
1043         struct bnad_rx_info *rx_info =
1044                         (struct bnad_rx_info *)ccb->cq->rx->priv;
1045
1046         rx_info->rx_ctrl[ccb->id].ccb = ccb;
1047         ccb->ctrl = &rx_info->rx_ctrl[ccb->id];
1048 }
1049
1050 static void
1051 bnad_cb_ccb_destroy(struct bnad *bnad, struct bna_ccb *ccb)
1052 {
1053         struct bnad_rx_info *rx_info =
1054                         (struct bnad_rx_info *)ccb->cq->rx->priv;
1055
1056         rx_info->rx_ctrl[ccb->id].ccb = NULL;
1057 }
1058
1059 static void
1060 bnad_cb_tx_stall(struct bnad *bnad, struct bna_tx *tx)
1061 {
1062         struct bnad_tx_info *tx_info =
1063                         (struct bnad_tx_info *)tx->priv;
1064         struct bna_tcb *tcb;
1065         u32 txq_id;
1066         int i;
1067
1068         for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) {
1069                 tcb = tx_info->tcb[i];
1070                 if (!tcb)
1071                         continue;
1072                 txq_id = tcb->id;
1073                 clear_bit(BNAD_TXQ_TX_STARTED, &tcb->flags);
1074                 netif_stop_subqueue(bnad->netdev, txq_id);
1075         }
1076 }
1077
1078 static void
1079 bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx)
1080 {
1081         struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv;
1082         struct bna_tcb *tcb;
1083         u32 txq_id;
1084         int i;
1085
1086         for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) {
1087                 tcb = tx_info->tcb[i];
1088                 if (!tcb)
1089                         continue;
1090                 txq_id = tcb->id;
1091
1092                 BUG_ON(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags));
1093                 set_bit(BNAD_TXQ_TX_STARTED, &tcb->flags);
1094                 BUG_ON(*(tcb->hw_consumer_index) != 0);
1095
1096                 if (netif_carrier_ok(bnad->netdev)) {
1097                         netif_wake_subqueue(bnad->netdev, txq_id);
1098                         BNAD_UPDATE_CTR(bnad, netif_queue_wakeup);
1099                 }
1100         }
1101
1102         /*
1103          * Workaround for first ioceth enable failure & we
1104          * get a 0 MAC address. We try to get the MAC address
1105          * again here.
1106          */
1107         if (is_zero_ether_addr(bnad->perm_addr)) {
1108                 bna_enet_perm_mac_get(&bnad->bna.enet, bnad->perm_addr);
1109                 bnad_set_netdev_perm_addr(bnad);
1110         }
1111 }
1112
1113 /*
1114  * Free all TxQs buffers and then notify TX_E_CLEANUP_DONE to Tx fsm.
1115  */
1116 static void
1117 bnad_tx_cleanup(struct delayed_work *work)
1118 {
1119         struct bnad_tx_info *tx_info =
1120                 container_of(work, struct bnad_tx_info, tx_cleanup_work);
1121         struct bnad *bnad = NULL;
1122         struct bna_tcb *tcb;
1123         unsigned long flags;
1124         u32 i, pending = 0;
1125
1126         for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) {
1127                 tcb = tx_info->tcb[i];
1128                 if (!tcb)
1129                         continue;
1130
1131                 bnad = tcb->bnad;
1132
1133                 if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) {
1134                         pending++;
1135                         continue;
1136                 }
1137
1138                 bnad_txq_cleanup(bnad, tcb);
1139
1140                 smp_mb__before_atomic();
1141                 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags);
1142         }
1143
1144         if (pending) {
1145                 queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work,
1146                         msecs_to_jiffies(1));
1147                 return;
1148         }
1149
1150         spin_lock_irqsave(&bnad->bna_lock, flags);
1151         bna_tx_cleanup_complete(tx_info->tx);
1152         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1153 }
1154
1155 static void
1156 bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx)
1157 {
1158         struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv;
1159         struct bna_tcb *tcb;
1160         int i;
1161
1162         for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) {
1163                 tcb = tx_info->tcb[i];
1164                 if (!tcb)
1165                         continue;
1166         }
1167
1168         queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 0);
1169 }
1170
1171 static void
1172 bnad_cb_rx_stall(struct bnad *bnad, struct bna_rx *rx)
1173 {
1174         struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv;
1175         struct bna_ccb *ccb;
1176         struct bnad_rx_ctrl *rx_ctrl;
1177         int i;
1178
1179         for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) {
1180                 rx_ctrl = &rx_info->rx_ctrl[i];
1181                 ccb = rx_ctrl->ccb;
1182                 if (!ccb)
1183                         continue;
1184
1185                 clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[0]->flags);
1186
1187                 if (ccb->rcb[1])
1188                         clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[1]->flags);
1189         }
1190 }
1191
1192 /*
1193  * Free all RxQs buffers and then notify RX_E_CLEANUP_DONE to Rx fsm.
1194  */
1195 static void
1196 bnad_rx_cleanup(void *work)
1197 {
1198         struct bnad_rx_info *rx_info =
1199                 container_of(work, struct bnad_rx_info, rx_cleanup_work);
1200         struct bnad_rx_ctrl *rx_ctrl;
1201         struct bnad *bnad = NULL;
1202         unsigned long flags;
1203         u32 i;
1204
1205         for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) {
1206                 rx_ctrl = &rx_info->rx_ctrl[i];
1207
1208                 if (!rx_ctrl->ccb)
1209                         continue;
1210
1211                 bnad = rx_ctrl->ccb->bnad;
1212
1213                 /*
1214                  * Wait till the poll handler has exited
1215                  * and nothing can be scheduled anymore
1216                  */
1217                 napi_disable(&rx_ctrl->napi);
1218
1219                 bnad_cq_cleanup(bnad, rx_ctrl->ccb);
1220                 bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[0]);
1221                 if (rx_ctrl->ccb->rcb[1])
1222                         bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[1]);
1223         }
1224
1225         spin_lock_irqsave(&bnad->bna_lock, flags);
1226         bna_rx_cleanup_complete(rx_info->rx);
1227         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1228 }
1229
1230 static void
1231 bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx)
1232 {
1233         struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv;
1234         struct bna_ccb *ccb;
1235         struct bnad_rx_ctrl *rx_ctrl;
1236         int i;
1237
1238         for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) {
1239                 rx_ctrl = &rx_info->rx_ctrl[i];
1240                 ccb = rx_ctrl->ccb;
1241                 if (!ccb)
1242                         continue;
1243
1244                 clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags);
1245
1246                 if (ccb->rcb[1])
1247                         clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[1]->flags);
1248         }
1249
1250         queue_work(bnad->work_q, &rx_info->rx_cleanup_work);
1251 }
1252
1253 static void
1254 bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx)
1255 {
1256         struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv;
1257         struct bna_ccb *ccb;
1258         struct bna_rcb *rcb;
1259         struct bnad_rx_ctrl *rx_ctrl;
1260         int i, j;
1261
1262         for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) {
1263                 rx_ctrl = &rx_info->rx_ctrl[i];
1264                 ccb = rx_ctrl->ccb;
1265                 if (!ccb)
1266                         continue;
1267
1268                 napi_enable(&rx_ctrl->napi);
1269
1270                 for (j = 0; j < BNAD_MAX_RXQ_PER_RXP; j++) {
1271                         rcb = ccb->rcb[j];
1272                         if (!rcb)
1273                                 continue;
1274
1275                         bnad_rxq_alloc_init(bnad, rcb);
1276                         set_bit(BNAD_RXQ_STARTED, &rcb->flags);
1277                         set_bit(BNAD_RXQ_POST_OK, &rcb->flags);
1278                         bnad_rxq_post(bnad, rcb);
1279                 }
1280         }
1281 }
1282
1283 static void
1284 bnad_cb_rx_disabled(void *arg, struct bna_rx *rx)
1285 {
1286         struct bnad *bnad = (struct bnad *)arg;
1287
1288         complete(&bnad->bnad_completions.rx_comp);
1289 }
1290
1291 static void
1292 bnad_cb_rx_mcast_add(struct bnad *bnad, struct bna_rx *rx)
1293 {
1294         bnad->bnad_completions.mcast_comp_status = BNA_CB_SUCCESS;
1295         complete(&bnad->bnad_completions.mcast_comp);
1296 }
1297
1298 void
1299 bnad_cb_stats_get(struct bnad *bnad, enum bna_cb_status status,
1300                        struct bna_stats *stats)
1301 {
1302         if (status == BNA_CB_SUCCESS)
1303                 BNAD_UPDATE_CTR(bnad, hw_stats_updates);
1304
1305         if (!netif_running(bnad->netdev) ||
1306                 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags))
1307                 return;
1308
1309         mod_timer(&bnad->stats_timer,
1310                   jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ));
1311 }
1312
1313 static void
1314 bnad_cb_enet_mtu_set(struct bnad *bnad)
1315 {
1316         bnad->bnad_completions.mtu_comp_status = BNA_CB_SUCCESS;
1317         complete(&bnad->bnad_completions.mtu_comp);
1318 }
1319
1320 void
1321 bnad_cb_completion(void *arg, enum bfa_status status)
1322 {
1323         struct bnad_iocmd_comp *iocmd_comp =
1324                         (struct bnad_iocmd_comp *)arg;
1325
1326         iocmd_comp->comp_status = (u32) status;
1327         complete(&iocmd_comp->comp);
1328 }
1329
1330 /* Resource allocation, free functions */
1331
1332 static void
1333 bnad_mem_free(struct bnad *bnad,
1334               struct bna_mem_info *mem_info)
1335 {
1336         int i;
1337         dma_addr_t dma_pa;
1338
1339         if (mem_info->mdl == NULL)
1340                 return;
1341
1342         for (i = 0; i < mem_info->num; i++) {
1343                 if (mem_info->mdl[i].kva != NULL) {
1344                         if (mem_info->mem_type == BNA_MEM_T_DMA) {
1345                                 BNA_GET_DMA_ADDR(&(mem_info->mdl[i].dma),
1346                                                 dma_pa);
1347                                 dma_free_coherent(&bnad->pcidev->dev,
1348                                                   mem_info->mdl[i].len,
1349                                                   mem_info->mdl[i].kva, dma_pa);
1350                         } else
1351                                 kfree(mem_info->mdl[i].kva);
1352                 }
1353         }
1354         kfree(mem_info->mdl);
1355         mem_info->mdl = NULL;
1356 }
1357
1358 static int
1359 bnad_mem_alloc(struct bnad *bnad,
1360                struct bna_mem_info *mem_info)
1361 {
1362         int i;
1363         dma_addr_t dma_pa;
1364
1365         if ((mem_info->num == 0) || (mem_info->len == 0)) {
1366                 mem_info->mdl = NULL;
1367                 return 0;
1368         }
1369
1370         mem_info->mdl = kcalloc(mem_info->num, sizeof(struct bna_mem_descr),
1371                                 GFP_KERNEL);
1372         if (mem_info->mdl == NULL)
1373                 return -ENOMEM;
1374
1375         if (mem_info->mem_type == BNA_MEM_T_DMA) {
1376                 for (i = 0; i < mem_info->num; i++) {
1377                         mem_info->mdl[i].len = mem_info->len;
1378                         mem_info->mdl[i].kva =
1379                                 dma_alloc_coherent(&bnad->pcidev->dev,
1380                                                    mem_info->len, &dma_pa,
1381                                                    GFP_KERNEL);
1382                         if (mem_info->mdl[i].kva == NULL)
1383                                 goto err_return;
1384
1385                         BNA_SET_DMA_ADDR(dma_pa,
1386                                          &(mem_info->mdl[i].dma));
1387                 }
1388         } else {
1389                 for (i = 0; i < mem_info->num; i++) {
1390                         mem_info->mdl[i].len = mem_info->len;
1391                         mem_info->mdl[i].kva = kzalloc(mem_info->len,
1392                                                         GFP_KERNEL);
1393                         if (mem_info->mdl[i].kva == NULL)
1394                                 goto err_return;
1395                 }
1396         }
1397
1398         return 0;
1399
1400 err_return:
1401         bnad_mem_free(bnad, mem_info);
1402         return -ENOMEM;
1403 }
1404
1405 /* Free IRQ for Mailbox */
1406 static void
1407 bnad_mbox_irq_free(struct bnad *bnad)
1408 {
1409         int irq;
1410         unsigned long flags;
1411
1412         spin_lock_irqsave(&bnad->bna_lock, flags);
1413         bnad_disable_mbox_irq(bnad);
1414         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1415
1416         irq = BNAD_GET_MBOX_IRQ(bnad);
1417         free_irq(irq, bnad);
1418 }
1419
1420 /*
1421  * Allocates IRQ for Mailbox, but keep it disabled
1422  * This will be enabled once we get the mbox enable callback
1423  * from bna
1424  */
1425 static int
1426 bnad_mbox_irq_alloc(struct bnad *bnad)
1427 {
1428         int             err = 0;
1429         unsigned long   irq_flags, flags;
1430         u32     irq;
1431         irq_handler_t   irq_handler;
1432
1433         spin_lock_irqsave(&bnad->bna_lock, flags);
1434         if (bnad->cfg_flags & BNAD_CF_MSIX) {
1435                 irq_handler = (irq_handler_t)bnad_msix_mbox_handler;
1436                 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector;
1437                 irq_flags = 0;
1438         } else {
1439                 irq_handler = (irq_handler_t)bnad_isr;
1440                 irq = bnad->pcidev->irq;
1441                 irq_flags = IRQF_SHARED;
1442         }
1443
1444         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1445         sprintf(bnad->mbox_irq_name, "%s", BNAD_NAME);
1446
1447         /*
1448          * Set the Mbox IRQ disable flag, so that the IRQ handler
1449          * called from request_irq() for SHARED IRQs do not execute
1450          */
1451         set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags);
1452
1453         BNAD_UPDATE_CTR(bnad, mbox_intr_disabled);
1454
1455         err = request_irq(irq, irq_handler, irq_flags,
1456                           bnad->mbox_irq_name, bnad);
1457
1458         return err;
1459 }
1460
1461 static void
1462 bnad_txrx_irq_free(struct bnad *bnad, struct bna_intr_info *intr_info)
1463 {
1464         kfree(intr_info->idl);
1465         intr_info->idl = NULL;
1466 }
1467
1468 /* Allocates Interrupt Descriptor List for MSIX/INT-X vectors */
1469 static int
1470 bnad_txrx_irq_alloc(struct bnad *bnad, enum bnad_intr_source src,
1471                     u32 txrx_id, struct bna_intr_info *intr_info)
1472 {
1473         int i, vector_start = 0;
1474         u32 cfg_flags;
1475         unsigned long flags;
1476
1477         spin_lock_irqsave(&bnad->bna_lock, flags);
1478         cfg_flags = bnad->cfg_flags;
1479         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1480
1481         if (cfg_flags & BNAD_CF_MSIX) {
1482                 intr_info->intr_type = BNA_INTR_T_MSIX;
1483                 intr_info->idl = kcalloc(intr_info->num,
1484                                         sizeof(struct bna_intr_descr),
1485                                         GFP_KERNEL);
1486                 if (!intr_info->idl)
1487                         return -ENOMEM;
1488
1489                 switch (src) {
1490                 case BNAD_INTR_TX:
1491                         vector_start = BNAD_MAILBOX_MSIX_VECTORS + txrx_id;
1492                         break;
1493
1494                 case BNAD_INTR_RX:
1495                         vector_start = BNAD_MAILBOX_MSIX_VECTORS +
1496                                         (bnad->num_tx * bnad->num_txq_per_tx) +
1497                                         txrx_id;
1498                         break;
1499
1500                 default:
1501                         BUG();
1502                 }
1503
1504                 for (i = 0; i < intr_info->num; i++)
1505                         intr_info->idl[i].vector = vector_start + i;
1506         } else {
1507                 intr_info->intr_type = BNA_INTR_T_INTX;
1508                 intr_info->num = 1;
1509                 intr_info->idl = kcalloc(intr_info->num,
1510                                         sizeof(struct bna_intr_descr),
1511                                         GFP_KERNEL);
1512                 if (!intr_info->idl)
1513                         return -ENOMEM;
1514
1515                 switch (src) {
1516                 case BNAD_INTR_TX:
1517                         intr_info->idl[0].vector = BNAD_INTX_TX_IB_BITMASK;
1518                         break;
1519
1520                 case BNAD_INTR_RX:
1521                         intr_info->idl[0].vector = BNAD_INTX_RX_IB_BITMASK;
1522                         break;
1523                 }
1524         }
1525         return 0;
1526 }
1527
1528 /* NOTE: Should be called for MSIX only
1529  * Unregisters Tx MSIX vector(s) from the kernel
1530  */
1531 static void
1532 bnad_tx_msix_unregister(struct bnad *bnad, struct bnad_tx_info *tx_info,
1533                         int num_txqs)
1534 {
1535         int i;
1536         int vector_num;
1537
1538         for (i = 0; i < num_txqs; i++) {
1539                 if (tx_info->tcb[i] == NULL)
1540                         continue;
1541
1542                 vector_num = tx_info->tcb[i]->intr_vector;
1543                 free_irq(bnad->msix_table[vector_num].vector, tx_info->tcb[i]);
1544         }
1545 }
1546
1547 /* NOTE: Should be called for MSIX only
1548  * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel
1549  */
1550 static int
1551 bnad_tx_msix_register(struct bnad *bnad, struct bnad_tx_info *tx_info,
1552                         u32 tx_id, int num_txqs)
1553 {
1554         int i;
1555         int err;
1556         int vector_num;
1557
1558         for (i = 0; i < num_txqs; i++) {
1559                 vector_num = tx_info->tcb[i]->intr_vector;
1560                 sprintf(tx_info->tcb[i]->name, "%s TXQ %d", bnad->netdev->name,
1561                                 tx_id + tx_info->tcb[i]->id);
1562                 err = request_irq(bnad->msix_table[vector_num].vector,
1563                                   (irq_handler_t)bnad_msix_tx, 0,
1564                                   tx_info->tcb[i]->name,
1565                                   tx_info->tcb[i]);
1566                 if (err)
1567                         goto err_return;
1568         }
1569
1570         return 0;
1571
1572 err_return:
1573         if (i > 0)
1574                 bnad_tx_msix_unregister(bnad, tx_info, (i - 1));
1575         return -1;
1576 }
1577
1578 /* NOTE: Should be called for MSIX only
1579  * Unregisters Rx MSIX vector(s) from the kernel
1580  */
1581 static void
1582 bnad_rx_msix_unregister(struct bnad *bnad, struct bnad_rx_info *rx_info,
1583                         int num_rxps)
1584 {
1585         int i;
1586         int vector_num;
1587
1588         for (i = 0; i < num_rxps; i++) {
1589                 if (rx_info->rx_ctrl[i].ccb == NULL)
1590                         continue;
1591
1592                 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector;
1593                 free_irq(bnad->msix_table[vector_num].vector,
1594                          rx_info->rx_ctrl[i].ccb);
1595         }
1596 }
1597
1598 /* NOTE: Should be called for MSIX only
1599  * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel
1600  */
1601 static int
1602 bnad_rx_msix_register(struct bnad *bnad, struct bnad_rx_info *rx_info,
1603                         u32 rx_id, int num_rxps)
1604 {
1605         int i;
1606         int err;
1607         int vector_num;
1608
1609         for (i = 0; i < num_rxps; i++) {
1610                 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector;
1611                 sprintf(rx_info->rx_ctrl[i].ccb->name, "%s CQ %d",
1612                         bnad->netdev->name,
1613                         rx_id + rx_info->rx_ctrl[i].ccb->id);
1614                 err = request_irq(bnad->msix_table[vector_num].vector,
1615                                   (irq_handler_t)bnad_msix_rx, 0,
1616                                   rx_info->rx_ctrl[i].ccb->name,
1617                                   rx_info->rx_ctrl[i].ccb);
1618                 if (err)
1619                         goto err_return;
1620         }
1621
1622         return 0;
1623
1624 err_return:
1625         if (i > 0)
1626                 bnad_rx_msix_unregister(bnad, rx_info, (i - 1));
1627         return -1;
1628 }
1629
1630 /* Free Tx object Resources */
1631 static void
1632 bnad_tx_res_free(struct bnad *bnad, struct bna_res_info *res_info)
1633 {
1634         int i;
1635
1636         for (i = 0; i < BNA_TX_RES_T_MAX; i++) {
1637                 if (res_info[i].res_type == BNA_RES_T_MEM)
1638                         bnad_mem_free(bnad, &res_info[i].res_u.mem_info);
1639                 else if (res_info[i].res_type == BNA_RES_T_INTR)
1640                         bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info);
1641         }
1642 }
1643
1644 /* Allocates memory and interrupt resources for Tx object */
1645 static int
1646 bnad_tx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info,
1647                   u32 tx_id)
1648 {
1649         int i, err = 0;
1650
1651         for (i = 0; i < BNA_TX_RES_T_MAX; i++) {
1652                 if (res_info[i].res_type == BNA_RES_T_MEM)
1653                         err = bnad_mem_alloc(bnad,
1654                                         &res_info[i].res_u.mem_info);
1655                 else if (res_info[i].res_type == BNA_RES_T_INTR)
1656                         err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_TX, tx_id,
1657                                         &res_info[i].res_u.intr_info);
1658                 if (err)
1659                         goto err_return;
1660         }
1661         return 0;
1662
1663 err_return:
1664         bnad_tx_res_free(bnad, res_info);
1665         return err;
1666 }
1667
1668 /* Free Rx object Resources */
1669 static void
1670 bnad_rx_res_free(struct bnad *bnad, struct bna_res_info *res_info)
1671 {
1672         int i;
1673
1674         for (i = 0; i < BNA_RX_RES_T_MAX; i++) {
1675                 if (res_info[i].res_type == BNA_RES_T_MEM)
1676                         bnad_mem_free(bnad, &res_info[i].res_u.mem_info);
1677                 else if (res_info[i].res_type == BNA_RES_T_INTR)
1678                         bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info);
1679         }
1680 }
1681
1682 /* Allocates memory and interrupt resources for Rx object */
1683 static int
1684 bnad_rx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info,
1685                   uint rx_id)
1686 {
1687         int i, err = 0;
1688
1689         /* All memory needs to be allocated before setup_ccbs */
1690         for (i = 0; i < BNA_RX_RES_T_MAX; i++) {
1691                 if (res_info[i].res_type == BNA_RES_T_MEM)
1692                         err = bnad_mem_alloc(bnad,
1693                                         &res_info[i].res_u.mem_info);
1694                 else if (res_info[i].res_type == BNA_RES_T_INTR)
1695                         err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_RX, rx_id,
1696                                         &res_info[i].res_u.intr_info);
1697                 if (err)
1698                         goto err_return;
1699         }
1700         return 0;
1701
1702 err_return:
1703         bnad_rx_res_free(bnad, res_info);
1704         return err;
1705 }
1706
1707 /* Timer callbacks */
1708 /* a) IOC timer */
1709 static void
1710 bnad_ioc_timeout(unsigned long data)
1711 {
1712         struct bnad *bnad = (struct bnad *)data;
1713         unsigned long flags;
1714
1715         spin_lock_irqsave(&bnad->bna_lock, flags);
1716         bfa_nw_ioc_timeout(&bnad->bna.ioceth.ioc);
1717         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1718 }
1719
1720 static void
1721 bnad_ioc_hb_check(unsigned long data)
1722 {
1723         struct bnad *bnad = (struct bnad *)data;
1724         unsigned long flags;
1725
1726         spin_lock_irqsave(&bnad->bna_lock, flags);
1727         bfa_nw_ioc_hb_check(&bnad->bna.ioceth.ioc);
1728         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1729 }
1730
1731 static void
1732 bnad_iocpf_timeout(unsigned long data)
1733 {
1734         struct bnad *bnad = (struct bnad *)data;
1735         unsigned long flags;
1736
1737         spin_lock_irqsave(&bnad->bna_lock, flags);
1738         bfa_nw_iocpf_timeout(&bnad->bna.ioceth.ioc);
1739         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1740 }
1741
1742 static void
1743 bnad_iocpf_sem_timeout(unsigned long data)
1744 {
1745         struct bnad *bnad = (struct bnad *)data;
1746         unsigned long flags;
1747
1748         spin_lock_irqsave(&bnad->bna_lock, flags);
1749         bfa_nw_iocpf_sem_timeout(&bnad->bna.ioceth.ioc);
1750         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1751 }
1752
1753 /*
1754  * All timer routines use bnad->bna_lock to protect against
1755  * the following race, which may occur in case of no locking:
1756  *      Time    CPU m   CPU n
1757  *      0       1 = test_bit
1758  *      1                       clear_bit
1759  *      2                       del_timer_sync
1760  *      3       mod_timer
1761  */
1762
1763 /* b) Dynamic Interrupt Moderation Timer */
1764 static void
1765 bnad_dim_timeout(unsigned long data)
1766 {
1767         struct bnad *bnad = (struct bnad *)data;
1768         struct bnad_rx_info *rx_info;
1769         struct bnad_rx_ctrl *rx_ctrl;
1770         int i, j;
1771         unsigned long flags;
1772
1773         if (!netif_carrier_ok(bnad->netdev))
1774                 return;
1775
1776         spin_lock_irqsave(&bnad->bna_lock, flags);
1777         for (i = 0; i < bnad->num_rx; i++) {
1778                 rx_info = &bnad->rx_info[i];
1779                 if (!rx_info->rx)
1780                         continue;
1781                 for (j = 0; j < bnad->num_rxp_per_rx; j++) {
1782                         rx_ctrl = &rx_info->rx_ctrl[j];
1783                         if (!rx_ctrl->ccb)
1784                                 continue;
1785                         bna_rx_dim_update(rx_ctrl->ccb);
1786                 }
1787         }
1788
1789         /* Check for BNAD_CF_DIM_ENABLED, does not eleminate a race */
1790         if (test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags))
1791                 mod_timer(&bnad->dim_timer,
1792                           jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ));
1793         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1794 }
1795
1796 /* c)  Statistics Timer */
1797 static void
1798 bnad_stats_timeout(unsigned long data)
1799 {
1800         struct bnad *bnad = (struct bnad *)data;
1801         unsigned long flags;
1802
1803         if (!netif_running(bnad->netdev) ||
1804                 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags))
1805                 return;
1806
1807         spin_lock_irqsave(&bnad->bna_lock, flags);
1808         bna_hw_stats_get(&bnad->bna);
1809         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1810 }
1811
1812 /*
1813  * Set up timer for DIM
1814  * Called with bnad->bna_lock held
1815  */
1816 void
1817 bnad_dim_timer_start(struct bnad *bnad)
1818 {
1819         if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED &&
1820             !test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) {
1821                 setup_timer(&bnad->dim_timer, bnad_dim_timeout,
1822                             (unsigned long)bnad);
1823                 set_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags);
1824                 mod_timer(&bnad->dim_timer,
1825                           jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ));
1826         }
1827 }
1828
1829 /*
1830  * Set up timer for statistics
1831  * Called with mutex_lock(&bnad->conf_mutex) held
1832  */
1833 static void
1834 bnad_stats_timer_start(struct bnad *bnad)
1835 {
1836         unsigned long flags;
1837
1838         spin_lock_irqsave(&bnad->bna_lock, flags);
1839         if (!test_and_set_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) {
1840                 setup_timer(&bnad->stats_timer, bnad_stats_timeout,
1841                             (unsigned long)bnad);
1842                 mod_timer(&bnad->stats_timer,
1843                           jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ));
1844         }
1845         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1846 }
1847
1848 /*
1849  * Stops the stats timer
1850  * Called with mutex_lock(&bnad->conf_mutex) held
1851  */
1852 static void
1853 bnad_stats_timer_stop(struct bnad *bnad)
1854 {
1855         int to_del = 0;
1856         unsigned long flags;
1857
1858         spin_lock_irqsave(&bnad->bna_lock, flags);
1859         if (test_and_clear_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags))
1860                 to_del = 1;
1861         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1862         if (to_del)
1863                 del_timer_sync(&bnad->stats_timer);
1864 }
1865
1866 /* Utilities */
1867
1868 static void
1869 bnad_netdev_mc_list_get(struct net_device *netdev, u8 *mc_list)
1870 {
1871         int i = 1; /* Index 0 has broadcast address */
1872         struct netdev_hw_addr *mc_addr;
1873
1874         netdev_for_each_mc_addr(mc_addr, netdev) {
1875                 ether_addr_copy(&mc_list[i * ETH_ALEN], &mc_addr->addr[0]);
1876                 i++;
1877         }
1878 }
1879
1880 static int
1881 bnad_napi_poll_rx(struct napi_struct *napi, int budget)
1882 {
1883         struct bnad_rx_ctrl *rx_ctrl =
1884                 container_of(napi, struct bnad_rx_ctrl, napi);
1885         struct bnad *bnad = rx_ctrl->bnad;
1886         int rcvd = 0;
1887
1888         rx_ctrl->rx_poll_ctr++;
1889
1890         if (!netif_carrier_ok(bnad->netdev))
1891                 goto poll_exit;
1892
1893         rcvd = bnad_cq_process(bnad, rx_ctrl->ccb, budget);
1894         if (rcvd >= budget)
1895                 return rcvd;
1896
1897 poll_exit:
1898         napi_complete(napi);
1899
1900         rx_ctrl->rx_complete++;
1901
1902         if (rx_ctrl->ccb)
1903                 bnad_enable_rx_irq_unsafe(rx_ctrl->ccb);
1904
1905         return rcvd;
1906 }
1907
1908 #define BNAD_NAPI_POLL_QUOTA            64
1909 static void
1910 bnad_napi_add(struct bnad *bnad, u32 rx_id)
1911 {
1912         struct bnad_rx_ctrl *rx_ctrl;
1913         int i;
1914
1915         /* Initialize & enable NAPI */
1916         for (i = 0; i < bnad->num_rxp_per_rx; i++) {
1917                 rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i];
1918                 netif_napi_add(bnad->netdev, &rx_ctrl->napi,
1919                                bnad_napi_poll_rx, BNAD_NAPI_POLL_QUOTA);
1920         }
1921 }
1922
1923 static void
1924 bnad_napi_delete(struct bnad *bnad, u32 rx_id)
1925 {
1926         int i;
1927
1928         /* First disable and then clean up */
1929         for (i = 0; i < bnad->num_rxp_per_rx; i++)
1930                 netif_napi_del(&bnad->rx_info[rx_id].rx_ctrl[i].napi);
1931 }
1932
1933 /* Should be held with conf_lock held */
1934 void
1935 bnad_destroy_tx(struct bnad *bnad, u32 tx_id)
1936 {
1937         struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id];
1938         struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0];
1939         unsigned long flags;
1940
1941         if (!tx_info->tx)
1942                 return;
1943
1944         init_completion(&bnad->bnad_completions.tx_comp);
1945         spin_lock_irqsave(&bnad->bna_lock, flags);
1946         bna_tx_disable(tx_info->tx, BNA_HARD_CLEANUP, bnad_cb_tx_disabled);
1947         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1948         wait_for_completion(&bnad->bnad_completions.tx_comp);
1949
1950         if (tx_info->tcb[0]->intr_type == BNA_INTR_T_MSIX)
1951                 bnad_tx_msix_unregister(bnad, tx_info,
1952                         bnad->num_txq_per_tx);
1953
1954         spin_lock_irqsave(&bnad->bna_lock, flags);
1955         bna_tx_destroy(tx_info->tx);
1956         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1957
1958         tx_info->tx = NULL;
1959         tx_info->tx_id = 0;
1960
1961         bnad_tx_res_free(bnad, res_info);
1962 }
1963
1964 /* Should be held with conf_lock held */
1965 int
1966 bnad_setup_tx(struct bnad *bnad, u32 tx_id)
1967 {
1968         int err;
1969         struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id];
1970         struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0];
1971         struct bna_intr_info *intr_info =
1972                         &res_info[BNA_TX_RES_INTR_T_TXCMPL].res_u.intr_info;
1973         struct bna_tx_config *tx_config = &bnad->tx_config[tx_id];
1974         static const struct bna_tx_event_cbfn tx_cbfn = {
1975                 .tcb_setup_cbfn = bnad_cb_tcb_setup,
1976                 .tcb_destroy_cbfn = bnad_cb_tcb_destroy,
1977                 .tx_stall_cbfn = bnad_cb_tx_stall,
1978                 .tx_resume_cbfn = bnad_cb_tx_resume,
1979                 .tx_cleanup_cbfn = bnad_cb_tx_cleanup,
1980         };
1981
1982         struct bna_tx *tx;
1983         unsigned long flags;
1984
1985         tx_info->tx_id = tx_id;
1986
1987         /* Initialize the Tx object configuration */
1988         tx_config->num_txq = bnad->num_txq_per_tx;
1989         tx_config->txq_depth = bnad->txq_depth;
1990         tx_config->tx_type = BNA_TX_T_REGULAR;
1991         tx_config->coalescing_timeo = bnad->tx_coalescing_timeo;
1992
1993         /* Get BNA's resource requirement for one tx object */
1994         spin_lock_irqsave(&bnad->bna_lock, flags);
1995         bna_tx_res_req(bnad->num_txq_per_tx,
1996                 bnad->txq_depth, res_info);
1997         spin_unlock_irqrestore(&bnad->bna_lock, flags);
1998
1999         /* Fill Unmap Q memory requirements */
2000         BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_TX_RES_MEM_T_UNMAPQ],
2001                         bnad->num_txq_per_tx, (sizeof(struct bnad_tx_unmap) *
2002                         bnad->txq_depth));
2003
2004         /* Allocate resources */
2005         err = bnad_tx_res_alloc(bnad, res_info, tx_id);
2006         if (err)
2007                 return err;
2008
2009         /* Ask BNA to create one Tx object, supplying required resources */
2010         spin_lock_irqsave(&bnad->bna_lock, flags);
2011         tx = bna_tx_create(&bnad->bna, bnad, tx_config, &tx_cbfn, res_info,
2012                         tx_info);
2013         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2014         if (!tx) {
2015                 err = -ENOMEM;
2016                 goto err_return;
2017         }
2018         tx_info->tx = tx;
2019
2020         INIT_DELAYED_WORK(&tx_info->tx_cleanup_work,
2021                         (work_func_t)bnad_tx_cleanup);
2022
2023         /* Register ISR for the Tx object */
2024         if (intr_info->intr_type == BNA_INTR_T_MSIX) {
2025                 err = bnad_tx_msix_register(bnad, tx_info,
2026                         tx_id, bnad->num_txq_per_tx);
2027                 if (err)
2028                         goto cleanup_tx;
2029         }
2030
2031         spin_lock_irqsave(&bnad->bna_lock, flags);
2032         bna_tx_enable(tx);
2033         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2034
2035         return 0;
2036
2037 cleanup_tx:
2038         spin_lock_irqsave(&bnad->bna_lock, flags);
2039         bna_tx_destroy(tx_info->tx);
2040         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2041         tx_info->tx = NULL;
2042         tx_info->tx_id = 0;
2043 err_return:
2044         bnad_tx_res_free(bnad, res_info);
2045         return err;
2046 }
2047
2048 /* Setup the rx config for bna_rx_create */
2049 /* bnad decides the configuration */
2050 static void
2051 bnad_init_rx_config(struct bnad *bnad, struct bna_rx_config *rx_config)
2052 {
2053         memset(rx_config, 0, sizeof(*rx_config));
2054         rx_config->rx_type = BNA_RX_T_REGULAR;
2055         rx_config->num_paths = bnad->num_rxp_per_rx;
2056         rx_config->coalescing_timeo = bnad->rx_coalescing_timeo;
2057
2058         if (bnad->num_rxp_per_rx > 1) {
2059                 rx_config->rss_status = BNA_STATUS_T_ENABLED;
2060                 rx_config->rss_config.hash_type =
2061                                 (BFI_ENET_RSS_IPV6 |
2062                                  BFI_ENET_RSS_IPV6_TCP |
2063                                  BFI_ENET_RSS_IPV4 |
2064                                  BFI_ENET_RSS_IPV4_TCP);
2065                 rx_config->rss_config.hash_mask =
2066                                 bnad->num_rxp_per_rx - 1;
2067                 netdev_rss_key_fill(rx_config->rss_config.toeplitz_hash_key,
2068                         sizeof(rx_config->rss_config.toeplitz_hash_key));
2069         } else {
2070                 rx_config->rss_status = BNA_STATUS_T_DISABLED;
2071                 memset(&rx_config->rss_config, 0,
2072                        sizeof(rx_config->rss_config));
2073         }
2074
2075         rx_config->frame_size = BNAD_FRAME_SIZE(bnad->netdev->mtu);
2076         rx_config->q0_multi_buf = BNA_STATUS_T_DISABLED;
2077
2078         /* BNA_RXP_SINGLE - one data-buffer queue
2079          * BNA_RXP_SLR - one small-buffer and one large-buffer queues
2080          * BNA_RXP_HDS - one header-buffer and one data-buffer queues
2081          */
2082         /* TODO: configurable param for queue type */
2083         rx_config->rxp_type = BNA_RXP_SLR;
2084
2085         if (BNAD_PCI_DEV_IS_CAT2(bnad) &&
2086             rx_config->frame_size > 4096) {
2087                 /* though size_routing_enable is set in SLR,
2088                  * small packets may get routed to same rxq.
2089                  * set buf_size to 2048 instead of PAGE_SIZE.
2090                  */
2091                 rx_config->q0_buf_size = 2048;
2092                 /* this should be in multiples of 2 */
2093                 rx_config->q0_num_vecs = 4;
2094                 rx_config->q0_depth = bnad->rxq_depth * rx_config->q0_num_vecs;
2095                 rx_config->q0_multi_buf = BNA_STATUS_T_ENABLED;
2096         } else {
2097                 rx_config->q0_buf_size = rx_config->frame_size;
2098                 rx_config->q0_num_vecs = 1;
2099                 rx_config->q0_depth = bnad->rxq_depth;
2100         }
2101
2102         /* initialize for q1 for BNA_RXP_SLR/BNA_RXP_HDS */
2103         if (rx_config->rxp_type == BNA_RXP_SLR) {
2104                 rx_config->q1_depth = bnad->rxq_depth;
2105                 rx_config->q1_buf_size = BFI_SMALL_RXBUF_SIZE;
2106         }
2107
2108         rx_config->vlan_strip_status =
2109                 (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) ?
2110                 BNA_STATUS_T_ENABLED : BNA_STATUS_T_DISABLED;
2111 }
2112
2113 static void
2114 bnad_rx_ctrl_init(struct bnad *bnad, u32 rx_id)
2115 {
2116         struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id];
2117         int i;
2118
2119         for (i = 0; i < bnad->num_rxp_per_rx; i++)
2120                 rx_info->rx_ctrl[i].bnad = bnad;
2121 }
2122
2123 /* Called with mutex_lock(&bnad->conf_mutex) held */
2124 static u32
2125 bnad_reinit_rx(struct bnad *bnad)
2126 {
2127         struct net_device *netdev = bnad->netdev;
2128         u32 err = 0, current_err = 0;
2129         u32 rx_id = 0, count = 0;
2130         unsigned long flags;
2131
2132         /* destroy and create new rx objects */
2133         for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) {
2134                 if (!bnad->rx_info[rx_id].rx)
2135                         continue;
2136                 bnad_destroy_rx(bnad, rx_id);
2137         }
2138
2139         spin_lock_irqsave(&bnad->bna_lock, flags);
2140         bna_enet_mtu_set(&bnad->bna.enet,
2141                          BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL);
2142         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2143
2144         for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) {
2145                 count++;
2146                 current_err = bnad_setup_rx(bnad, rx_id);
2147                 if (current_err && !err) {
2148                         err = current_err;
2149                         netdev_err(netdev, "RXQ:%u setup failed\n", rx_id);
2150                 }
2151         }
2152
2153         /* restore rx configuration */
2154         if (bnad->rx_info[0].rx && !err) {
2155                 bnad_restore_vlans(bnad, 0);
2156                 bnad_enable_default_bcast(bnad);
2157                 spin_lock_irqsave(&bnad->bna_lock, flags);
2158                 bnad_mac_addr_set_locked(bnad, netdev->dev_addr);
2159                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2160                 bnad_set_rx_mode(netdev);
2161         }
2162
2163         return count;
2164 }
2165
2166 /* Called with bnad_conf_lock() held */
2167 void
2168 bnad_destroy_rx(struct bnad *bnad, u32 rx_id)
2169 {
2170         struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id];
2171         struct bna_rx_config *rx_config = &bnad->rx_config[rx_id];
2172         struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0];
2173         unsigned long flags;
2174         int to_del = 0;
2175
2176         if (!rx_info->rx)
2177                 return;
2178
2179         if (0 == rx_id) {
2180                 spin_lock_irqsave(&bnad->bna_lock, flags);
2181                 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED &&
2182                     test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) {
2183                         clear_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags);
2184                         to_del = 1;
2185                 }
2186                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2187                 if (to_del)
2188                         del_timer_sync(&bnad->dim_timer);
2189         }
2190
2191         init_completion(&bnad->bnad_completions.rx_comp);
2192         spin_lock_irqsave(&bnad->bna_lock, flags);
2193         bna_rx_disable(rx_info->rx, BNA_HARD_CLEANUP, bnad_cb_rx_disabled);
2194         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2195         wait_for_completion(&bnad->bnad_completions.rx_comp);
2196
2197         if (rx_info->rx_ctrl[0].ccb->intr_type == BNA_INTR_T_MSIX)
2198                 bnad_rx_msix_unregister(bnad, rx_info, rx_config->num_paths);
2199
2200         bnad_napi_delete(bnad, rx_id);
2201
2202         spin_lock_irqsave(&bnad->bna_lock, flags);
2203         bna_rx_destroy(rx_info->rx);
2204
2205         rx_info->rx = NULL;
2206         rx_info->rx_id = 0;
2207         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2208
2209         bnad_rx_res_free(bnad, res_info);
2210 }
2211
2212 /* Called with mutex_lock(&bnad->conf_mutex) held */
2213 int
2214 bnad_setup_rx(struct bnad *bnad, u32 rx_id)
2215 {
2216         int err;
2217         struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id];
2218         struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0];
2219         struct bna_intr_info *intr_info =
2220                         &res_info[BNA_RX_RES_T_INTR].res_u.intr_info;
2221         struct bna_rx_config *rx_config = &bnad->rx_config[rx_id];
2222         static const struct bna_rx_event_cbfn rx_cbfn = {
2223                 .rcb_setup_cbfn = NULL,
2224                 .rcb_destroy_cbfn = NULL,
2225                 .ccb_setup_cbfn = bnad_cb_ccb_setup,
2226                 .ccb_destroy_cbfn = bnad_cb_ccb_destroy,
2227                 .rx_stall_cbfn = bnad_cb_rx_stall,
2228                 .rx_cleanup_cbfn = bnad_cb_rx_cleanup,
2229                 .rx_post_cbfn = bnad_cb_rx_post,
2230         };
2231         struct bna_rx *rx;
2232         unsigned long flags;
2233
2234         rx_info->rx_id = rx_id;
2235
2236         /* Initialize the Rx object configuration */
2237         bnad_init_rx_config(bnad, rx_config);
2238
2239         /* Get BNA's resource requirement for one Rx object */
2240         spin_lock_irqsave(&bnad->bna_lock, flags);
2241         bna_rx_res_req(rx_config, res_info);
2242         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2243
2244         /* Fill Unmap Q memory requirements */
2245         BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPDQ],
2246                                  rx_config->num_paths,
2247                         (rx_config->q0_depth *
2248                          sizeof(struct bnad_rx_unmap)) +
2249                          sizeof(struct bnad_rx_unmap_q));
2250
2251         if (rx_config->rxp_type != BNA_RXP_SINGLE) {
2252                 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPHQ],
2253                                          rx_config->num_paths,
2254                                 (rx_config->q1_depth *
2255                                  sizeof(struct bnad_rx_unmap) +
2256                                  sizeof(struct bnad_rx_unmap_q)));
2257         }
2258         /* Allocate resource */
2259         err = bnad_rx_res_alloc(bnad, res_info, rx_id);
2260         if (err)
2261                 return err;
2262
2263         bnad_rx_ctrl_init(bnad, rx_id);
2264
2265         /* Ask BNA to create one Rx object, supplying required resources */
2266         spin_lock_irqsave(&bnad->bna_lock, flags);
2267         rx = bna_rx_create(&bnad->bna, bnad, rx_config, &rx_cbfn, res_info,
2268                         rx_info);
2269         if (!rx) {
2270                 err = -ENOMEM;
2271                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2272                 goto err_return;
2273         }
2274         rx_info->rx = rx;
2275         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2276
2277         INIT_WORK(&rx_info->rx_cleanup_work,
2278                         (work_func_t)(bnad_rx_cleanup));
2279
2280         /*
2281          * Init NAPI, so that state is set to NAPI_STATE_SCHED,
2282          * so that IRQ handler cannot schedule NAPI at this point.
2283          */
2284         bnad_napi_add(bnad, rx_id);
2285
2286         /* Register ISR for the Rx object */
2287         if (intr_info->intr_type == BNA_INTR_T_MSIX) {
2288                 err = bnad_rx_msix_register(bnad, rx_info, rx_id,
2289                                                 rx_config->num_paths);
2290                 if (err)
2291                         goto err_return;
2292         }
2293
2294         spin_lock_irqsave(&bnad->bna_lock, flags);
2295         if (0 == rx_id) {
2296                 /* Set up Dynamic Interrupt Moderation Vector */
2297                 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED)
2298                         bna_rx_dim_reconfig(&bnad->bna, bna_napi_dim_vector);
2299
2300                 /* Enable VLAN filtering only on the default Rx */
2301                 bna_rx_vlanfilter_enable(rx);
2302
2303                 /* Start the DIM timer */
2304                 bnad_dim_timer_start(bnad);
2305         }
2306
2307         bna_rx_enable(rx);
2308         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2309
2310         return 0;
2311
2312 err_return:
2313         bnad_destroy_rx(bnad, rx_id);
2314         return err;
2315 }
2316
2317 /* Called with conf_lock & bnad->bna_lock held */
2318 void
2319 bnad_tx_coalescing_timeo_set(struct bnad *bnad)
2320 {
2321         struct bnad_tx_info *tx_info;
2322
2323         tx_info = &bnad->tx_info[0];
2324         if (!tx_info->tx)
2325                 return;
2326
2327         bna_tx_coalescing_timeo_set(tx_info->tx, bnad->tx_coalescing_timeo);
2328 }
2329
2330 /* Called with conf_lock & bnad->bna_lock held */
2331 void
2332 bnad_rx_coalescing_timeo_set(struct bnad *bnad)
2333 {
2334         struct bnad_rx_info *rx_info;
2335         int     i;
2336
2337         for (i = 0; i < bnad->num_rx; i++) {
2338                 rx_info = &bnad->rx_info[i];
2339                 if (!rx_info->rx)
2340                         continue;
2341                 bna_rx_coalescing_timeo_set(rx_info->rx,
2342                                 bnad->rx_coalescing_timeo);
2343         }
2344 }
2345
2346 /*
2347  * Called with bnad->bna_lock held
2348  */
2349 int
2350 bnad_mac_addr_set_locked(struct bnad *bnad, const u8 *mac_addr)
2351 {
2352         int ret;
2353
2354         if (!is_valid_ether_addr(mac_addr))
2355                 return -EADDRNOTAVAIL;
2356
2357         /* If datapath is down, pretend everything went through */
2358         if (!bnad->rx_info[0].rx)
2359                 return 0;
2360
2361         ret = bna_rx_ucast_set(bnad->rx_info[0].rx, mac_addr);
2362         if (ret != BNA_CB_SUCCESS)
2363                 return -EADDRNOTAVAIL;
2364
2365         return 0;
2366 }
2367
2368 /* Should be called with conf_lock held */
2369 int
2370 bnad_enable_default_bcast(struct bnad *bnad)
2371 {
2372         struct bnad_rx_info *rx_info = &bnad->rx_info[0];
2373         int ret;
2374         unsigned long flags;
2375
2376         init_completion(&bnad->bnad_completions.mcast_comp);
2377
2378         spin_lock_irqsave(&bnad->bna_lock, flags);
2379         ret = bna_rx_mcast_add(rx_info->rx, bnad_bcast_addr,
2380                                bnad_cb_rx_mcast_add);
2381         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2382
2383         if (ret == BNA_CB_SUCCESS)
2384                 wait_for_completion(&bnad->bnad_completions.mcast_comp);
2385         else
2386                 return -ENODEV;
2387
2388         if (bnad->bnad_completions.mcast_comp_status != BNA_CB_SUCCESS)
2389                 return -ENODEV;
2390
2391         return 0;
2392 }
2393
2394 /* Called with mutex_lock(&bnad->conf_mutex) held */
2395 void
2396 bnad_restore_vlans(struct bnad *bnad, u32 rx_id)
2397 {
2398         u16 vid;
2399         unsigned long flags;
2400
2401         for_each_set_bit(vid, bnad->active_vlans, VLAN_N_VID) {
2402                 spin_lock_irqsave(&bnad->bna_lock, flags);
2403                 bna_rx_vlan_add(bnad->rx_info[rx_id].rx, vid);
2404                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2405         }
2406 }
2407
2408 /* Statistics utilities */
2409 void
2410 bnad_netdev_qstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats)
2411 {
2412         int i, j;
2413
2414         for (i = 0; i < bnad->num_rx; i++) {
2415                 for (j = 0; j < bnad->num_rxp_per_rx; j++) {
2416                         if (bnad->rx_info[i].rx_ctrl[j].ccb) {
2417                                 stats->rx_packets += bnad->rx_info[i].
2418                                 rx_ctrl[j].ccb->rcb[0]->rxq->rx_packets;
2419                                 stats->rx_bytes += bnad->rx_info[i].
2420                                         rx_ctrl[j].ccb->rcb[0]->rxq->rx_bytes;
2421                                 if (bnad->rx_info[i].rx_ctrl[j].ccb->rcb[1] &&
2422                                         bnad->rx_info[i].rx_ctrl[j].ccb->
2423                                         rcb[1]->rxq) {
2424                                         stats->rx_packets +=
2425                                                 bnad->rx_info[i].rx_ctrl[j].
2426                                                 ccb->rcb[1]->rxq->rx_packets;
2427                                         stats->rx_bytes +=
2428                                                 bnad->rx_info[i].rx_ctrl[j].
2429                                                 ccb->rcb[1]->rxq->rx_bytes;
2430                                 }
2431                         }
2432                 }
2433         }
2434         for (i = 0; i < bnad->num_tx; i++) {
2435                 for (j = 0; j < bnad->num_txq_per_tx; j++) {
2436                         if (bnad->tx_info[i].tcb[j]) {
2437                                 stats->tx_packets +=
2438                                 bnad->tx_info[i].tcb[j]->txq->tx_packets;
2439                                 stats->tx_bytes +=
2440                                         bnad->tx_info[i].tcb[j]->txq->tx_bytes;
2441                         }
2442                 }
2443         }
2444 }
2445
2446 /*
2447  * Must be called with the bna_lock held.
2448  */
2449 void
2450 bnad_netdev_hwstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats)
2451 {
2452         struct bfi_enet_stats_mac *mac_stats;
2453         u32 bmap;
2454         int i;
2455
2456         mac_stats = &bnad->stats.bna_stats->hw_stats.mac_stats;
2457         stats->rx_errors =
2458                 mac_stats->rx_fcs_error + mac_stats->rx_alignment_error +
2459                 mac_stats->rx_frame_length_error + mac_stats->rx_code_error +
2460                 mac_stats->rx_undersize;
2461         stats->tx_errors = mac_stats->tx_fcs_error +
2462                                         mac_stats->tx_undersize;
2463         stats->rx_dropped = mac_stats->rx_drop;
2464         stats->tx_dropped = mac_stats->tx_drop;
2465         stats->multicast = mac_stats->rx_multicast;
2466         stats->collisions = mac_stats->tx_total_collision;
2467
2468         stats->rx_length_errors = mac_stats->rx_frame_length_error;
2469
2470         /* receive ring buffer overflow  ?? */
2471
2472         stats->rx_crc_errors = mac_stats->rx_fcs_error;
2473         stats->rx_frame_errors = mac_stats->rx_alignment_error;
2474         /* recv'r fifo overrun */
2475         bmap = bna_rx_rid_mask(&bnad->bna);
2476         for (i = 0; bmap; i++) {
2477                 if (bmap & 1) {
2478                         stats->rx_fifo_errors +=
2479                                 bnad->stats.bna_stats->
2480                                         hw_stats.rxf_stats[i].frame_drops;
2481                         break;
2482                 }
2483                 bmap >>= 1;
2484         }
2485 }
2486
2487 static void
2488 bnad_mbox_irq_sync(struct bnad *bnad)
2489 {
2490         u32 irq;
2491         unsigned long flags;
2492
2493         spin_lock_irqsave(&bnad->bna_lock, flags);
2494         if (bnad->cfg_flags & BNAD_CF_MSIX)
2495                 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector;
2496         else
2497                 irq = bnad->pcidev->irq;
2498         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2499
2500         synchronize_irq(irq);
2501 }
2502
2503 /* Utility used by bnad_start_xmit, for doing TSO */
2504 static int
2505 bnad_tso_prepare(struct bnad *bnad, struct sk_buff *skb)
2506 {
2507         int err;
2508
2509         err = skb_cow_head(skb, 0);
2510         if (err < 0) {
2511                 BNAD_UPDATE_CTR(bnad, tso_err);
2512                 return err;
2513         }
2514
2515         /*
2516          * For TSO, the TCP checksum field is seeded with pseudo-header sum
2517          * excluding the length field.
2518          */
2519         if (vlan_get_protocol(skb) == htons(ETH_P_IP)) {
2520                 struct iphdr *iph = ip_hdr(skb);
2521
2522                 /* Do we really need these? */
2523                 iph->tot_len = 0;
2524                 iph->check = 0;
2525
2526                 tcp_hdr(skb)->check =
2527                         ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
2528                                            IPPROTO_TCP, 0);
2529                 BNAD_UPDATE_CTR(bnad, tso4);
2530         } else {
2531                 struct ipv6hdr *ipv6h = ipv6_hdr(skb);
2532
2533                 ipv6h->payload_len = 0;
2534                 tcp_hdr(skb)->check =
2535                         ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr, 0,
2536                                          IPPROTO_TCP, 0);
2537                 BNAD_UPDATE_CTR(bnad, tso6);
2538         }
2539
2540         return 0;
2541 }
2542
2543 /*
2544  * Initialize Q numbers depending on Rx Paths
2545  * Called with bnad->bna_lock held, because of cfg_flags
2546  * access.
2547  */
2548 static void
2549 bnad_q_num_init(struct bnad *bnad)
2550 {
2551         int rxps;
2552
2553         rxps = min((uint)num_online_cpus(),
2554                         (uint)(BNAD_MAX_RX * BNAD_MAX_RXP_PER_RX));
2555
2556         if (!(bnad->cfg_flags & BNAD_CF_MSIX))
2557                 rxps = 1;       /* INTx */
2558
2559         bnad->num_rx = 1;
2560         bnad->num_tx = 1;
2561         bnad->num_rxp_per_rx = rxps;
2562         bnad->num_txq_per_tx = BNAD_TXQ_NUM;
2563 }
2564
2565 /*
2566  * Adjusts the Q numbers, given a number of msix vectors
2567  * Give preference to RSS as opposed to Tx priority Queues,
2568  * in such a case, just use 1 Tx Q
2569  * Called with bnad->bna_lock held b'cos of cfg_flags access
2570  */
2571 static void
2572 bnad_q_num_adjust(struct bnad *bnad, int msix_vectors, int temp)
2573 {
2574         bnad->num_txq_per_tx = 1;
2575         if ((msix_vectors >= (bnad->num_tx * bnad->num_txq_per_tx)  +
2576              bnad_rxqs_per_cq + BNAD_MAILBOX_MSIX_VECTORS) &&
2577             (bnad->cfg_flags & BNAD_CF_MSIX)) {
2578                 bnad->num_rxp_per_rx = msix_vectors -
2579                         (bnad->num_tx * bnad->num_txq_per_tx) -
2580                         BNAD_MAILBOX_MSIX_VECTORS;
2581         } else
2582                 bnad->num_rxp_per_rx = 1;
2583 }
2584
2585 /* Enable / disable ioceth */
2586 static int
2587 bnad_ioceth_disable(struct bnad *bnad)
2588 {
2589         unsigned long flags;
2590         int err = 0;
2591
2592         spin_lock_irqsave(&bnad->bna_lock, flags);
2593         init_completion(&bnad->bnad_completions.ioc_comp);
2594         bna_ioceth_disable(&bnad->bna.ioceth, BNA_HARD_CLEANUP);
2595         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2596
2597         wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp,
2598                 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT));
2599
2600         err = bnad->bnad_completions.ioc_comp_status;
2601         return err;
2602 }
2603
2604 static int
2605 bnad_ioceth_enable(struct bnad *bnad)
2606 {
2607         int err = 0;
2608         unsigned long flags;
2609
2610         spin_lock_irqsave(&bnad->bna_lock, flags);
2611         init_completion(&bnad->bnad_completions.ioc_comp);
2612         bnad->bnad_completions.ioc_comp_status = BNA_CB_WAITING;
2613         bna_ioceth_enable(&bnad->bna.ioceth);
2614         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2615
2616         wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp,
2617                 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT));
2618
2619         err = bnad->bnad_completions.ioc_comp_status;
2620
2621         return err;
2622 }
2623
2624 /* Free BNA resources */
2625 static void
2626 bnad_res_free(struct bnad *bnad, struct bna_res_info *res_info,
2627                 u32 res_val_max)
2628 {
2629         int i;
2630
2631         for (i = 0; i < res_val_max; i++)
2632                 bnad_mem_free(bnad, &res_info[i].res_u.mem_info);
2633 }
2634
2635 /* Allocates memory and interrupt resources for BNA */
2636 static int
2637 bnad_res_alloc(struct bnad *bnad, struct bna_res_info *res_info,
2638                 u32 res_val_max)
2639 {
2640         int i, err;
2641
2642         for (i = 0; i < res_val_max; i++) {
2643                 err = bnad_mem_alloc(bnad, &res_info[i].res_u.mem_info);
2644                 if (err)
2645                         goto err_return;
2646         }
2647         return 0;
2648
2649 err_return:
2650         bnad_res_free(bnad, res_info, res_val_max);
2651         return err;
2652 }
2653
2654 /* Interrupt enable / disable */
2655 static void
2656 bnad_enable_msix(struct bnad *bnad)
2657 {
2658         int i, ret;
2659         unsigned long flags;
2660
2661         spin_lock_irqsave(&bnad->bna_lock, flags);
2662         if (!(bnad->cfg_flags & BNAD_CF_MSIX)) {
2663                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2664                 return;
2665         }
2666         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2667
2668         if (bnad->msix_table)
2669                 return;
2670
2671         bnad->msix_table =
2672                 kcalloc(bnad->msix_num, sizeof(struct msix_entry), GFP_KERNEL);
2673
2674         if (!bnad->msix_table)
2675                 goto intx_mode;
2676
2677         for (i = 0; i < bnad->msix_num; i++)
2678                 bnad->msix_table[i].entry = i;
2679
2680         ret = pci_enable_msix_range(bnad->pcidev, bnad->msix_table,
2681                                     1, bnad->msix_num);
2682         if (ret < 0) {
2683                 goto intx_mode;
2684         } else if (ret < bnad->msix_num) {
2685                 dev_warn(&bnad->pcidev->dev,
2686                          "%d MSI-X vectors allocated < %d requested\n",
2687                          ret, bnad->msix_num);
2688
2689                 spin_lock_irqsave(&bnad->bna_lock, flags);
2690                 /* ret = #of vectors that we got */
2691                 bnad_q_num_adjust(bnad, (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2,
2692                         (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2);
2693                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
2694
2695                 bnad->msix_num = BNAD_NUM_TXQ + BNAD_NUM_RXP +
2696                          BNAD_MAILBOX_MSIX_VECTORS;
2697
2698                 if (bnad->msix_num > ret) {
2699                         pci_disable_msix(bnad->pcidev);
2700                         goto intx_mode;
2701                 }
2702         }
2703
2704         pci_intx(bnad->pcidev, 0);
2705
2706         return;
2707
2708 intx_mode:
2709         dev_warn(&bnad->pcidev->dev,
2710                  "MSI-X enable failed - operating in INTx mode\n");
2711
2712         kfree(bnad->msix_table);
2713         bnad->msix_table = NULL;
2714         bnad->msix_num = 0;
2715         spin_lock_irqsave(&bnad->bna_lock, flags);
2716         bnad->cfg_flags &= ~BNAD_CF_MSIX;
2717         bnad_q_num_init(bnad);
2718         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2719 }
2720
2721 static void
2722 bnad_disable_msix(struct bnad *bnad)
2723 {
2724         u32 cfg_flags;
2725         unsigned long flags;
2726
2727         spin_lock_irqsave(&bnad->bna_lock, flags);
2728         cfg_flags = bnad->cfg_flags;
2729         if (bnad->cfg_flags & BNAD_CF_MSIX)
2730                 bnad->cfg_flags &= ~BNAD_CF_MSIX;
2731         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2732
2733         if (cfg_flags & BNAD_CF_MSIX) {
2734                 pci_disable_msix(bnad->pcidev);
2735                 kfree(bnad->msix_table);
2736                 bnad->msix_table = NULL;
2737         }
2738 }
2739
2740 /* Netdev entry points */
2741 static int
2742 bnad_open(struct net_device *netdev)
2743 {
2744         int err;
2745         struct bnad *bnad = netdev_priv(netdev);
2746         struct bna_pause_config pause_config;
2747         unsigned long flags;
2748
2749         mutex_lock(&bnad->conf_mutex);
2750
2751         /* Tx */
2752         err = bnad_setup_tx(bnad, 0);
2753         if (err)
2754                 goto err_return;
2755
2756         /* Rx */
2757         err = bnad_setup_rx(bnad, 0);
2758         if (err)
2759                 goto cleanup_tx;
2760
2761         /* Port */
2762         pause_config.tx_pause = 0;
2763         pause_config.rx_pause = 0;
2764
2765         spin_lock_irqsave(&bnad->bna_lock, flags);
2766         bna_enet_mtu_set(&bnad->bna.enet,
2767                          BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL);
2768         bna_enet_pause_config(&bnad->bna.enet, &pause_config);
2769         bna_enet_enable(&bnad->bna.enet);
2770         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2771
2772         /* Enable broadcast */
2773         bnad_enable_default_bcast(bnad);
2774
2775         /* Restore VLANs, if any */
2776         bnad_restore_vlans(bnad, 0);
2777
2778         /* Set the UCAST address */
2779         spin_lock_irqsave(&bnad->bna_lock, flags);
2780         bnad_mac_addr_set_locked(bnad, netdev->dev_addr);
2781         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2782
2783         /* Start the stats timer */
2784         bnad_stats_timer_start(bnad);
2785
2786         mutex_unlock(&bnad->conf_mutex);
2787
2788         return 0;
2789
2790 cleanup_tx:
2791         bnad_destroy_tx(bnad, 0);
2792
2793 err_return:
2794         mutex_unlock(&bnad->conf_mutex);
2795         return err;
2796 }
2797
2798 static int
2799 bnad_stop(struct net_device *netdev)
2800 {
2801         struct bnad *bnad = netdev_priv(netdev);
2802         unsigned long flags;
2803
2804         mutex_lock(&bnad->conf_mutex);
2805
2806         /* Stop the stats timer */
2807         bnad_stats_timer_stop(bnad);
2808
2809         init_completion(&bnad->bnad_completions.enet_comp);
2810
2811         spin_lock_irqsave(&bnad->bna_lock, flags);
2812         bna_enet_disable(&bnad->bna.enet, BNA_HARD_CLEANUP,
2813                         bnad_cb_enet_disabled);
2814         spin_unlock_irqrestore(&bnad->bna_lock, flags);
2815
2816         wait_for_completion(&bnad->bnad_completions.enet_comp);
2817
2818         bnad_destroy_tx(bnad, 0);
2819         bnad_destroy_rx(bnad, 0);
2820
2821         /* Synchronize mailbox IRQ */
2822         bnad_mbox_irq_sync(bnad);
2823
2824         mutex_unlock(&bnad->conf_mutex);
2825
2826         return 0;
2827 }
2828
2829 /* TX */
2830 /* Returns 0 for success */
2831 static int
2832 bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb,
2833                     struct sk_buff *skb, struct bna_txq_entry *txqent)
2834 {
2835         u16 flags = 0;
2836         u32 gso_size;
2837         u16 vlan_tag = 0;
2838
2839         if (skb_vlan_tag_present(skb)) {
2840                 vlan_tag = (u16)skb_vlan_tag_get(skb);
2841                 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN);
2842         }
2843         if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) {
2844                 vlan_tag = ((tcb->priority & 0x7) << VLAN_PRIO_SHIFT)
2845                                 | (vlan_tag & 0x1fff);
2846                 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN);
2847         }
2848         txqent->hdr.wi.vlan_tag = htons(vlan_tag);
2849
2850         if (skb_is_gso(skb)) {
2851                 gso_size = skb_shinfo(skb)->gso_size;
2852                 if (unlikely(gso_size > bnad->netdev->mtu)) {
2853                         BNAD_UPDATE_CTR(bnad, tx_skb_mss_too_long);
2854                         return -EINVAL;
2855                 }
2856                 if (unlikely((gso_size + skb_transport_offset(skb) +
2857                               tcp_hdrlen(skb)) >= skb->len)) {
2858                         txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND);
2859                         txqent->hdr.wi.lso_mss = 0;
2860                         BNAD_UPDATE_CTR(bnad, tx_skb_tso_too_short);
2861                 } else {
2862                         txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND_LSO);
2863                         txqent->hdr.wi.lso_mss = htons(gso_size);
2864                 }
2865
2866                 if (bnad_tso_prepare(bnad, skb)) {
2867                         BNAD_UPDATE_CTR(bnad, tx_skb_tso_prepare);
2868                         return -EINVAL;
2869                 }
2870
2871                 flags |= (BNA_TXQ_WI_CF_IP_CKSUM | BNA_TXQ_WI_CF_TCP_CKSUM);
2872                 txqent->hdr.wi.l4_hdr_size_n_offset =
2873                         htons(BNA_TXQ_WI_L4_HDR_N_OFFSET(
2874                         tcp_hdrlen(skb) >> 2, skb_transport_offset(skb)));
2875         } else  {
2876                 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND);
2877                 txqent->hdr.wi.lso_mss = 0;
2878
2879                 if (unlikely(skb->len > (bnad->netdev->mtu + VLAN_ETH_HLEN))) {
2880                         BNAD_UPDATE_CTR(bnad, tx_skb_non_tso_too_long);
2881                         return -EINVAL;
2882                 }
2883
2884                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2885                         __be16 net_proto = vlan_get_protocol(skb);
2886                         u8 proto = 0;
2887
2888                         if (net_proto == htons(ETH_P_IP))
2889                                 proto = ip_hdr(skb)->protocol;
2890 #ifdef NETIF_F_IPV6_CSUM
2891                         else if (net_proto == htons(ETH_P_IPV6)) {
2892                                 /* nexthdr may not be TCP immediately. */
2893                                 proto = ipv6_hdr(skb)->nexthdr;
2894                         }
2895 #endif
2896                         if (proto == IPPROTO_TCP) {
2897                                 flags |= BNA_TXQ_WI_CF_TCP_CKSUM;
2898                                 txqent->hdr.wi.l4_hdr_size_n_offset =
2899                                         htons(BNA_TXQ_WI_L4_HDR_N_OFFSET
2900                                               (0, skb_transport_offset(skb)));
2901
2902                                 BNAD_UPDATE_CTR(bnad, tcpcsum_offload);
2903
2904                                 if (unlikely(skb_headlen(skb) <
2905                                             skb_transport_offset(skb) +
2906                                     tcp_hdrlen(skb))) {
2907                                         BNAD_UPDATE_CTR(bnad, tx_skb_tcp_hdr);
2908                                         return -EINVAL;
2909                                 }
2910                         } else if (proto == IPPROTO_UDP) {
2911                                 flags |= BNA_TXQ_WI_CF_UDP_CKSUM;
2912                                 txqent->hdr.wi.l4_hdr_size_n_offset =
2913                                         htons(BNA_TXQ_WI_L4_HDR_N_OFFSET
2914                                               (0, skb_transport_offset(skb)));
2915
2916                                 BNAD_UPDATE_CTR(bnad, udpcsum_offload);
2917                                 if (unlikely(skb_headlen(skb) <
2918                                             skb_transport_offset(skb) +
2919                                     sizeof(struct udphdr))) {
2920                                         BNAD_UPDATE_CTR(bnad, tx_skb_udp_hdr);
2921                                         return -EINVAL;
2922                                 }
2923                         } else {
2924
2925                                 BNAD_UPDATE_CTR(bnad, tx_skb_csum_err);
2926                                 return -EINVAL;
2927                         }
2928                 } else
2929                         txqent->hdr.wi.l4_hdr_size_n_offset = 0;
2930         }
2931
2932         txqent->hdr.wi.flags = htons(flags);
2933         txqent->hdr.wi.frame_length = htonl(skb->len);
2934
2935         return 0;
2936 }
2937
2938 /*
2939  * bnad_start_xmit : Netdev entry point for Transmit
2940  *                   Called under lock held by net_device
2941  */
2942 static netdev_tx_t
2943 bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev)
2944 {
2945         struct bnad *bnad = netdev_priv(netdev);
2946         u32 txq_id = 0;
2947         struct bna_tcb *tcb = NULL;
2948         struct bnad_tx_unmap *unmap_q, *unmap, *head_unmap;
2949         u32             prod, q_depth, vect_id;
2950         u32             wis, vectors, len;
2951         int             i;
2952         dma_addr_t              dma_addr;
2953         struct bna_txq_entry *txqent;
2954
2955         len = skb_headlen(skb);
2956
2957         /* Sanity checks for the skb */
2958
2959         if (unlikely(skb->len <= ETH_HLEN)) {
2960                 dev_kfree_skb_any(skb);
2961                 BNAD_UPDATE_CTR(bnad, tx_skb_too_short);
2962                 return NETDEV_TX_OK;
2963         }
2964         if (unlikely(len > BFI_TX_MAX_DATA_PER_VECTOR)) {
2965                 dev_kfree_skb_any(skb);
2966                 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero);
2967                 return NETDEV_TX_OK;
2968         }
2969         if (unlikely(len == 0)) {
2970                 dev_kfree_skb_any(skb);
2971                 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero);
2972                 return NETDEV_TX_OK;
2973         }
2974
2975         tcb = bnad->tx_info[0].tcb[txq_id];
2976
2977         /*
2978          * Takes care of the Tx that is scheduled between clearing the flag
2979          * and the netif_tx_stop_all_queues() call.
2980          */
2981         if (unlikely(!tcb || !test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) {
2982                 dev_kfree_skb_any(skb);
2983                 BNAD_UPDATE_CTR(bnad, tx_skb_stopping);
2984                 return NETDEV_TX_OK;
2985         }
2986
2987         q_depth = tcb->q_depth;
2988         prod = tcb->producer_index;
2989         unmap_q = tcb->unmap_q;
2990
2991         vectors = 1 + skb_shinfo(skb)->nr_frags;
2992         wis = BNA_TXQ_WI_NEEDED(vectors);       /* 4 vectors per work item */
2993
2994         if (unlikely(vectors > BFI_TX_MAX_VECTORS_PER_PKT)) {
2995                 dev_kfree_skb_any(skb);
2996                 BNAD_UPDATE_CTR(bnad, tx_skb_max_vectors);
2997                 return NETDEV_TX_OK;
2998         }
2999
3000         /* Check for available TxQ resources */
3001         if (unlikely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) {
3002                 if ((*tcb->hw_consumer_index != tcb->consumer_index) &&
3003                     !test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) {
3004                         u32 sent;
3005                         sent = bnad_txcmpl_process(bnad, tcb);
3006                         if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)))
3007                                 bna_ib_ack(tcb->i_dbell, sent);
3008                         smp_mb__before_atomic();
3009                         clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags);
3010                 } else {
3011                         netif_stop_queue(netdev);
3012                         BNAD_UPDATE_CTR(bnad, netif_queue_stop);
3013                 }
3014
3015                 smp_mb();
3016                 /*
3017                  * Check again to deal with race condition between
3018                  * netif_stop_queue here, and netif_wake_queue in
3019                  * interrupt handler which is not inside netif tx lock.
3020                  */
3021                 if (likely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) {
3022                         BNAD_UPDATE_CTR(bnad, netif_queue_stop);
3023                         return NETDEV_TX_BUSY;
3024                 } else {
3025                         netif_wake_queue(netdev);
3026                         BNAD_UPDATE_CTR(bnad, netif_queue_wakeup);
3027                 }
3028         }
3029
3030         txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod];
3031         head_unmap = &unmap_q[prod];
3032
3033         /* Program the opcode, flags, frame_len, num_vectors in WI */
3034         if (bnad_txq_wi_prepare(bnad, tcb, skb, txqent)) {
3035                 dev_kfree_skb_any(skb);
3036                 return NETDEV_TX_OK;
3037         }
3038         txqent->hdr.wi.reserved = 0;
3039         txqent->hdr.wi.num_vectors = vectors;
3040
3041         head_unmap->skb = skb;
3042         head_unmap->nvecs = 0;
3043
3044         /* Program the vectors */
3045         unmap = head_unmap;
3046         dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data,
3047                                   len, DMA_TO_DEVICE);
3048         if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
3049                 dev_kfree_skb_any(skb);
3050                 BNAD_UPDATE_CTR(bnad, tx_skb_map_failed);
3051                 return NETDEV_TX_OK;
3052         }
3053         BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr);
3054         txqent->vector[0].length = htons(len);
3055         dma_unmap_addr_set(&unmap->vectors[0], dma_addr, dma_addr);
3056         head_unmap->nvecs++;
3057
3058         for (i = 0, vect_id = 0; i < vectors - 1; i++) {
3059                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
3060                 u32             size = skb_frag_size(frag);
3061
3062                 if (unlikely(size == 0)) {
3063                         /* Undo the changes starting at tcb->producer_index */
3064                         bnad_tx_buff_unmap(bnad, unmap_q, q_depth,
3065                                 tcb->producer_index);
3066                         dev_kfree_skb_any(skb);
3067                         BNAD_UPDATE_CTR(bnad, tx_skb_frag_zero);
3068                         return NETDEV_TX_OK;
3069                 }
3070
3071                 len += size;
3072
3073                 vect_id++;
3074                 if (vect_id == BFI_TX_MAX_VECTORS_PER_WI) {
3075                         vect_id = 0;
3076                         BNA_QE_INDX_INC(prod, q_depth);
3077                         txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod];
3078                         txqent->hdr.wi_ext.opcode = htons(BNA_TXQ_WI_EXTENSION);
3079                         unmap = &unmap_q[prod];
3080                 }
3081
3082                 dma_addr = skb_frag_dma_map(&bnad->pcidev->dev, frag,
3083                                             0, size, DMA_TO_DEVICE);
3084                 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) {
3085                         /* Undo the changes starting at tcb->producer_index */
3086                         bnad_tx_buff_unmap(bnad, unmap_q, q_depth,
3087                                            tcb->producer_index);
3088                         dev_kfree_skb_any(skb);
3089                         BNAD_UPDATE_CTR(bnad, tx_skb_map_failed);
3090                         return NETDEV_TX_OK;
3091                 }
3092
3093                 dma_unmap_len_set(&unmap->vectors[vect_id], dma_len, size);
3094                 BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[vect_id].host_addr);
3095                 txqent->vector[vect_id].length = htons(size);
3096                 dma_unmap_addr_set(&unmap->vectors[vect_id], dma_addr,
3097                                    dma_addr);
3098                 head_unmap->nvecs++;
3099         }
3100
3101         if (unlikely(len != skb->len)) {
3102                 /* Undo the changes starting at tcb->producer_index */
3103                 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, tcb->producer_index);
3104                 dev_kfree_skb_any(skb);
3105                 BNAD_UPDATE_CTR(bnad, tx_skb_len_mismatch);
3106                 return NETDEV_TX_OK;
3107         }
3108
3109         BNA_QE_INDX_INC(prod, q_depth);
3110         tcb->producer_index = prod;
3111
3112         smp_mb();
3113
3114         if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)))
3115                 return NETDEV_TX_OK;
3116
3117         skb_tx_timestamp(skb);
3118
3119         bna_txq_prod_indx_doorbell(tcb);
3120         smp_mb();
3121
3122         return NETDEV_TX_OK;
3123 }
3124
3125 /*
3126  * Used spin_lock to synchronize reading of stats structures, which
3127  * is written by BNA under the same lock.
3128  */
3129 static struct rtnl_link_stats64 *
3130 bnad_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
3131 {
3132         struct bnad *bnad = netdev_priv(netdev);
3133         unsigned long flags;
3134
3135         spin_lock_irqsave(&bnad->bna_lock, flags);
3136
3137         bnad_netdev_qstats_fill(bnad, stats);
3138         bnad_netdev_hwstats_fill(bnad, stats);
3139
3140         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3141
3142         return stats;
3143 }
3144
3145 static void
3146 bnad_set_rx_ucast_fltr(struct bnad *bnad)
3147 {
3148         struct net_device *netdev = bnad->netdev;
3149         int uc_count = netdev_uc_count(netdev);
3150         enum bna_cb_status ret;
3151         u8 *mac_list;
3152         struct netdev_hw_addr *ha;
3153         int entry;
3154
3155         if (netdev_uc_empty(bnad->netdev)) {
3156                 bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL);
3157                 return;
3158         }
3159
3160         if (uc_count > bna_attr(&bnad->bna)->num_ucmac)
3161                 goto mode_default;
3162
3163         mac_list = kzalloc(uc_count * ETH_ALEN, GFP_ATOMIC);
3164         if (mac_list == NULL)
3165                 goto mode_default;
3166
3167         entry = 0;
3168         netdev_for_each_uc_addr(ha, netdev) {
3169                 ether_addr_copy(&mac_list[entry * ETH_ALEN], &ha->addr[0]);
3170                 entry++;
3171         }
3172
3173         ret = bna_rx_ucast_listset(bnad->rx_info[0].rx, entry, mac_list);
3174         kfree(mac_list);
3175
3176         if (ret != BNA_CB_SUCCESS)
3177                 goto mode_default;
3178
3179         return;
3180
3181         /* ucast packets not in UCAM are routed to default function */
3182 mode_default:
3183         bnad->cfg_flags |= BNAD_CF_DEFAULT;
3184         bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL);
3185 }
3186
3187 static void
3188 bnad_set_rx_mcast_fltr(struct bnad *bnad)
3189 {
3190         struct net_device *netdev = bnad->netdev;
3191         int mc_count = netdev_mc_count(netdev);
3192         enum bna_cb_status ret;
3193         u8 *mac_list;
3194
3195         if (netdev->flags & IFF_ALLMULTI)
3196                 goto mode_allmulti;
3197
3198         if (netdev_mc_empty(netdev))
3199                 return;
3200
3201         if (mc_count > bna_attr(&bnad->bna)->num_mcmac)
3202                 goto mode_allmulti;
3203
3204         mac_list = kzalloc((mc_count + 1) * ETH_ALEN, GFP_ATOMIC);
3205
3206         if (mac_list == NULL)
3207                 goto mode_allmulti;
3208
3209         ether_addr_copy(&mac_list[0], &bnad_bcast_addr[0]);
3210
3211         /* copy rest of the MCAST addresses */
3212         bnad_netdev_mc_list_get(netdev, mac_list);
3213         ret = bna_rx_mcast_listset(bnad->rx_info[0].rx, mc_count + 1, mac_list);
3214         kfree(mac_list);
3215
3216         if (ret != BNA_CB_SUCCESS)
3217                 goto mode_allmulti;
3218
3219         return;
3220
3221 mode_allmulti:
3222         bnad->cfg_flags |= BNAD_CF_ALLMULTI;
3223         bna_rx_mcast_delall(bnad->rx_info[0].rx);
3224 }
3225
3226 void
3227 bnad_set_rx_mode(struct net_device *netdev)
3228 {
3229         struct bnad *bnad = netdev_priv(netdev);
3230         enum bna_rxmode new_mode, mode_mask;
3231         unsigned long flags;
3232
3233         spin_lock_irqsave(&bnad->bna_lock, flags);
3234
3235         if (bnad->rx_info[0].rx == NULL) {
3236                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
3237                 return;
3238         }
3239
3240         /* clear bnad flags to update it with new settings */
3241         bnad->cfg_flags &= ~(BNAD_CF_PROMISC | BNAD_CF_DEFAULT |
3242                         BNAD_CF_ALLMULTI);
3243
3244         new_mode = 0;
3245         if (netdev->flags & IFF_PROMISC) {
3246                 new_mode |= BNAD_RXMODE_PROMISC_DEFAULT;
3247                 bnad->cfg_flags |= BNAD_CF_PROMISC;
3248         } else {
3249                 bnad_set_rx_mcast_fltr(bnad);
3250
3251                 if (bnad->cfg_flags & BNAD_CF_ALLMULTI)
3252                         new_mode |= BNA_RXMODE_ALLMULTI;
3253
3254                 bnad_set_rx_ucast_fltr(bnad);
3255
3256                 if (bnad->cfg_flags & BNAD_CF_DEFAULT)
3257                         new_mode |= BNA_RXMODE_DEFAULT;
3258         }
3259
3260         mode_mask = BNA_RXMODE_PROMISC | BNA_RXMODE_DEFAULT |
3261                         BNA_RXMODE_ALLMULTI;
3262         bna_rx_mode_set(bnad->rx_info[0].rx, new_mode, mode_mask);
3263
3264         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3265 }
3266
3267 /*
3268  * bna_lock is used to sync writes to netdev->addr
3269  * conf_lock cannot be used since this call may be made
3270  * in a non-blocking context.
3271  */
3272 static int
3273 bnad_set_mac_address(struct net_device *netdev, void *addr)
3274 {
3275         int err;
3276         struct bnad *bnad = netdev_priv(netdev);
3277         struct sockaddr *sa = (struct sockaddr *)addr;
3278         unsigned long flags;
3279
3280         spin_lock_irqsave(&bnad->bna_lock, flags);
3281
3282         err = bnad_mac_addr_set_locked(bnad, sa->sa_data);
3283         if (!err)
3284                 ether_addr_copy(netdev->dev_addr, sa->sa_data);
3285
3286         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3287
3288         return err;
3289 }
3290
3291 static int
3292 bnad_mtu_set(struct bnad *bnad, int frame_size)
3293 {
3294         unsigned long flags;
3295
3296         init_completion(&bnad->bnad_completions.mtu_comp);
3297
3298         spin_lock_irqsave(&bnad->bna_lock, flags);
3299         bna_enet_mtu_set(&bnad->bna.enet, frame_size, bnad_cb_enet_mtu_set);
3300         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3301
3302         wait_for_completion(&bnad->bnad_completions.mtu_comp);
3303
3304         return bnad->bnad_completions.mtu_comp_status;
3305 }
3306
3307 static int
3308 bnad_change_mtu(struct net_device *netdev, int new_mtu)
3309 {
3310         int err, mtu;
3311         struct bnad *bnad = netdev_priv(netdev);
3312         u32 rx_count = 0, frame, new_frame;
3313
3314         if (new_mtu + ETH_HLEN < ETH_ZLEN || new_mtu > BNAD_JUMBO_MTU)
3315                 return -EINVAL;
3316
3317         mutex_lock(&bnad->conf_mutex);
3318
3319         mtu = netdev->mtu;
3320         netdev->mtu = new_mtu;
3321
3322         frame = BNAD_FRAME_SIZE(mtu);
3323         new_frame = BNAD_FRAME_SIZE(new_mtu);
3324
3325         /* check if multi-buffer needs to be enabled */
3326         if (BNAD_PCI_DEV_IS_CAT2(bnad) &&
3327             netif_running(bnad->netdev)) {
3328                 /* only when transition is over 4K */
3329                 if ((frame <= 4096 && new_frame > 4096) ||
3330                     (frame > 4096 && new_frame <= 4096))
3331                         rx_count = bnad_reinit_rx(bnad);
3332         }
3333
3334         /* rx_count > 0 - new rx created
3335          *      - Linux set err = 0 and return
3336          */
3337         err = bnad_mtu_set(bnad, new_frame);
3338         if (err)
3339                 err = -EBUSY;
3340
3341         mutex_unlock(&bnad->conf_mutex);
3342         return err;
3343 }
3344
3345 static int
3346 bnad_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
3347 {
3348         struct bnad *bnad = netdev_priv(netdev);
3349         unsigned long flags;
3350
3351         if (!bnad->rx_info[0].rx)
3352                 return 0;
3353
3354         mutex_lock(&bnad->conf_mutex);
3355
3356         spin_lock_irqsave(&bnad->bna_lock, flags);
3357         bna_rx_vlan_add(bnad->rx_info[0].rx, vid);
3358         set_bit(vid, bnad->active_vlans);
3359         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3360
3361         mutex_unlock(&bnad->conf_mutex);
3362
3363         return 0;
3364 }
3365
3366 static int
3367 bnad_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
3368 {
3369         struct bnad *bnad = netdev_priv(netdev);
3370         unsigned long flags;
3371
3372         if (!bnad->rx_info[0].rx)
3373                 return 0;
3374
3375         mutex_lock(&bnad->conf_mutex);
3376
3377         spin_lock_irqsave(&bnad->bna_lock, flags);
3378         clear_bit(vid, bnad->active_vlans);
3379         bna_rx_vlan_del(bnad->rx_info[0].rx, vid);
3380         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3381
3382         mutex_unlock(&bnad->conf_mutex);
3383
3384         return 0;
3385 }
3386
3387 static int bnad_set_features(struct net_device *dev, netdev_features_t features)
3388 {
3389         struct bnad *bnad = netdev_priv(dev);
3390         netdev_features_t changed = features ^ dev->features;
3391
3392         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(dev)) {
3393                 unsigned long flags;
3394
3395                 spin_lock_irqsave(&bnad->bna_lock, flags);
3396
3397                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3398                         bna_rx_vlan_strip_enable(bnad->rx_info[0].rx);
3399                 else
3400                         bna_rx_vlan_strip_disable(bnad->rx_info[0].rx);
3401
3402                 spin_unlock_irqrestore(&bnad->bna_lock, flags);
3403         }
3404
3405         return 0;
3406 }
3407
3408 #ifdef CONFIG_NET_POLL_CONTROLLER
3409 static void
3410 bnad_netpoll(struct net_device *netdev)
3411 {
3412         struct bnad *bnad = netdev_priv(netdev);
3413         struct bnad_rx_info *rx_info;
3414         struct bnad_rx_ctrl *rx_ctrl;
3415         u32 curr_mask;
3416         int i, j;
3417
3418         if (!(bnad->cfg_flags & BNAD_CF_MSIX)) {
3419                 bna_intx_disable(&bnad->bna, curr_mask);
3420                 bnad_isr(bnad->pcidev->irq, netdev);
3421                 bna_intx_enable(&bnad->bna, curr_mask);
3422         } else {
3423                 /*
3424                  * Tx processing may happen in sending context, so no need
3425                  * to explicitly process completions here
3426                  */
3427
3428                 /* Rx processing */
3429                 for (i = 0; i < bnad->num_rx; i++) {
3430                         rx_info = &bnad->rx_info[i];
3431                         if (!rx_info->rx)
3432                                 continue;
3433                         for (j = 0; j < bnad->num_rxp_per_rx; j++) {
3434                                 rx_ctrl = &rx_info->rx_ctrl[j];
3435                                 if (rx_ctrl->ccb)
3436                                         bnad_netif_rx_schedule_poll(bnad,
3437                                                             rx_ctrl->ccb);
3438                         }
3439                 }
3440         }
3441 }
3442 #endif
3443
3444 static const struct net_device_ops bnad_netdev_ops = {
3445         .ndo_open               = bnad_open,
3446         .ndo_stop               = bnad_stop,
3447         .ndo_start_xmit         = bnad_start_xmit,
3448         .ndo_get_stats64                = bnad_get_stats64,
3449         .ndo_set_rx_mode        = bnad_set_rx_mode,
3450         .ndo_validate_addr      = eth_validate_addr,
3451         .ndo_set_mac_address    = bnad_set_mac_address,
3452         .ndo_change_mtu         = bnad_change_mtu,
3453         .ndo_vlan_rx_add_vid    = bnad_vlan_rx_add_vid,
3454         .ndo_vlan_rx_kill_vid   = bnad_vlan_rx_kill_vid,
3455         .ndo_set_features       = bnad_set_features,
3456 #ifdef CONFIG_NET_POLL_CONTROLLER
3457         .ndo_poll_controller    = bnad_netpoll
3458 #endif
3459 };
3460
3461 static void
3462 bnad_netdev_init(struct bnad *bnad, bool using_dac)
3463 {
3464         struct net_device *netdev = bnad->netdev;
3465
3466         netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
3467                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3468                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_TX |
3469                 NETIF_F_HW_VLAN_CTAG_RX;
3470
3471         netdev->vlan_features = NETIF_F_SG | NETIF_F_HIGHDMA |
3472                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3473                 NETIF_F_TSO | NETIF_F_TSO6;
3474
3475         netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
3476
3477         if (using_dac)
3478                 netdev->features |= NETIF_F_HIGHDMA;
3479
3480         netdev->mem_start = bnad->mmio_start;
3481         netdev->mem_end = bnad->mmio_start + bnad->mmio_len - 1;
3482
3483         netdev->netdev_ops = &bnad_netdev_ops;
3484         bnad_set_ethtool_ops(netdev);
3485 }
3486
3487 /*
3488  * 1. Initialize the bnad structure
3489  * 2. Setup netdev pointer in pci_dev
3490  * 3. Initialize no. of TxQ & CQs & MSIX vectors
3491  * 4. Initialize work queue.
3492  */
3493 static int
3494 bnad_init(struct bnad *bnad,
3495           struct pci_dev *pdev, struct net_device *netdev)
3496 {
3497         unsigned long flags;
3498
3499         SET_NETDEV_DEV(netdev, &pdev->dev);
3500         pci_set_drvdata(pdev, netdev);
3501
3502         bnad->netdev = netdev;
3503         bnad->pcidev = pdev;
3504         bnad->mmio_start = pci_resource_start(pdev, 0);
3505         bnad->mmio_len = pci_resource_len(pdev, 0);
3506         bnad->bar0 = ioremap_nocache(bnad->mmio_start, bnad->mmio_len);
3507         if (!bnad->bar0) {
3508                 dev_err(&pdev->dev, "ioremap for bar0 failed\n");
3509                 return -ENOMEM;
3510         }
3511         dev_info(&pdev->dev, "bar0 mapped to %p, len %llu\n", bnad->bar0,
3512                  (unsigned long long) bnad->mmio_len);
3513
3514         spin_lock_irqsave(&bnad->bna_lock, flags);
3515         if (!bnad_msix_disable)
3516                 bnad->cfg_flags = BNAD_CF_MSIX;
3517
3518         bnad->cfg_flags |= BNAD_CF_DIM_ENABLED;
3519
3520         bnad_q_num_init(bnad);
3521         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3522
3523         bnad->msix_num = (bnad->num_tx * bnad->num_txq_per_tx) +
3524                 (bnad->num_rx * bnad->num_rxp_per_rx) +
3525                          BNAD_MAILBOX_MSIX_VECTORS;
3526
3527         bnad->txq_depth = BNAD_TXQ_DEPTH;
3528         bnad->rxq_depth = BNAD_RXQ_DEPTH;
3529
3530         bnad->tx_coalescing_timeo = BFI_TX_COALESCING_TIMEO;
3531         bnad->rx_coalescing_timeo = BFI_RX_COALESCING_TIMEO;
3532
3533         sprintf(bnad->wq_name, "%s_wq_%d", BNAD_NAME, bnad->id);
3534         bnad->work_q = create_singlethread_workqueue(bnad->wq_name);
3535         if (!bnad->work_q) {
3536                 iounmap(bnad->bar0);
3537                 return -ENOMEM;
3538         }
3539
3540         return 0;
3541 }
3542
3543 /*
3544  * Must be called after bnad_pci_uninit()
3545  * so that iounmap() and pci_set_drvdata(NULL)
3546  * happens only after PCI uninitialization.
3547  */
3548 static void
3549 bnad_uninit(struct bnad *bnad)
3550 {
3551         if (bnad->work_q) {
3552                 flush_workqueue(bnad->work_q);
3553                 destroy_workqueue(bnad->work_q);
3554                 bnad->work_q = NULL;
3555         }
3556
3557         if (bnad->bar0)
3558                 iounmap(bnad->bar0);
3559 }
3560
3561 /*
3562  * Initialize locks
3563         a) Per ioceth mutes used for serializing configuration
3564            changes from OS interface
3565         b) spin lock used to protect bna state machine
3566  */
3567 static void
3568 bnad_lock_init(struct bnad *bnad)
3569 {
3570         spin_lock_init(&bnad->bna_lock);
3571         mutex_init(&bnad->conf_mutex);
3572         mutex_init(&bnad_list_mutex);
3573 }
3574
3575 static void
3576 bnad_lock_uninit(struct bnad *bnad)
3577 {
3578         mutex_destroy(&bnad->conf_mutex);
3579         mutex_destroy(&bnad_list_mutex);
3580 }
3581
3582 /* PCI Initialization */
3583 static int
3584 bnad_pci_init(struct bnad *bnad,
3585               struct pci_dev *pdev, bool *using_dac)
3586 {
3587         int err;
3588
3589         err = pci_enable_device(pdev);
3590         if (err)
3591                 return err;
3592         err = pci_request_regions(pdev, BNAD_NAME);
3593         if (err)
3594                 goto disable_device;
3595         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
3596                 *using_dac = true;
3597         } else {
3598                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3599                 if (err)
3600                         goto release_regions;
3601                 *using_dac = false;
3602         }
3603         pci_set_master(pdev);
3604         return 0;
3605
3606 release_regions:
3607         pci_release_regions(pdev);
3608 disable_device:
3609         pci_disable_device(pdev);
3610
3611         return err;
3612 }
3613
3614 static void
3615 bnad_pci_uninit(struct pci_dev *pdev)
3616 {
3617         pci_release_regions(pdev);
3618         pci_disable_device(pdev);
3619 }
3620
3621 static int
3622 bnad_pci_probe(struct pci_dev *pdev,
3623                 const struct pci_device_id *pcidev_id)
3624 {
3625         bool    using_dac;
3626         int     err;
3627         struct bnad *bnad;
3628         struct bna *bna;
3629         struct net_device *netdev;
3630         struct bfa_pcidev pcidev_info;
3631         unsigned long flags;
3632
3633         mutex_lock(&bnad_fwimg_mutex);
3634         if (!cna_get_firmware_buf(pdev)) {
3635                 mutex_unlock(&bnad_fwimg_mutex);
3636                 dev_err(&pdev->dev, "failed to load firmware image!\n");
3637                 return -ENODEV;
3638         }
3639         mutex_unlock(&bnad_fwimg_mutex);
3640
3641         /*
3642          * Allocates sizeof(struct net_device + struct bnad)
3643          * bnad = netdev->priv
3644          */
3645         netdev = alloc_etherdev(sizeof(struct bnad));
3646         if (!netdev) {
3647                 err = -ENOMEM;
3648                 return err;
3649         }
3650         bnad = netdev_priv(netdev);
3651         bnad_lock_init(bnad);
3652         bnad_add_to_list(bnad);
3653         bnad->id = atomic_inc_return(&bna_id) - 1;
3654
3655         mutex_lock(&bnad->conf_mutex);
3656         /*
3657          * PCI initialization
3658          *      Output : using_dac = 1 for 64 bit DMA
3659          *                         = 0 for 32 bit DMA
3660          */
3661         using_dac = false;
3662         err = bnad_pci_init(bnad, pdev, &using_dac);
3663         if (err)
3664                 goto unlock_mutex;
3665
3666         /*
3667          * Initialize bnad structure
3668          * Setup relation between pci_dev & netdev
3669          */
3670         err = bnad_init(bnad, pdev, netdev);
3671         if (err)
3672                 goto pci_uninit;
3673
3674         /* Initialize netdev structure, set up ethtool ops */
3675         bnad_netdev_init(bnad, using_dac);
3676
3677         /* Set link to down state */
3678         netif_carrier_off(netdev);
3679
3680         /* Setup the debugfs node for this bfad */
3681         if (bna_debugfs_enable)
3682                 bnad_debugfs_init(bnad);
3683
3684         /* Get resource requirement form bna */
3685         spin_lock_irqsave(&bnad->bna_lock, flags);
3686         bna_res_req(&bnad->res_info[0]);
3687         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3688
3689         /* Allocate resources from bna */
3690         err = bnad_res_alloc(bnad, &bnad->res_info[0], BNA_RES_T_MAX);
3691         if (err)
3692                 goto drv_uninit;
3693
3694         bna = &bnad->bna;
3695
3696         /* Setup pcidev_info for bna_init() */
3697         pcidev_info.pci_slot = PCI_SLOT(bnad->pcidev->devfn);
3698         pcidev_info.pci_func = PCI_FUNC(bnad->pcidev->devfn);
3699         pcidev_info.device_id = bnad->pcidev->device;
3700         pcidev_info.pci_bar_kva = bnad->bar0;
3701
3702         spin_lock_irqsave(&bnad->bna_lock, flags);
3703         bna_init(bna, bnad, &pcidev_info, &bnad->res_info[0]);
3704         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3705
3706         bnad->stats.bna_stats = &bna->stats;
3707
3708         bnad_enable_msix(bnad);
3709         err = bnad_mbox_irq_alloc(bnad);
3710         if (err)
3711                 goto res_free;
3712
3713         /* Set up timers */
3714         setup_timer(&bnad->bna.ioceth.ioc.ioc_timer, bnad_ioc_timeout,
3715                     (unsigned long)bnad);
3716         setup_timer(&bnad->bna.ioceth.ioc.hb_timer, bnad_ioc_hb_check,
3717                     (unsigned long)bnad);
3718         setup_timer(&bnad->bna.ioceth.ioc.iocpf_timer, bnad_iocpf_timeout,
3719                     (unsigned long)bnad);
3720         setup_timer(&bnad->bna.ioceth.ioc.sem_timer, bnad_iocpf_sem_timeout,
3721                     (unsigned long)bnad);
3722
3723         /*
3724          * Start the chip
3725          * If the call back comes with error, we bail out.
3726          * This is a catastrophic error.
3727          */
3728         err = bnad_ioceth_enable(bnad);
3729         if (err) {
3730                 dev_err(&pdev->dev, "initialization failed err=%d\n", err);
3731                 goto probe_success;
3732         }
3733
3734         spin_lock_irqsave(&bnad->bna_lock, flags);
3735         if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) ||
3736                 bna_num_rxp_set(bna, BNAD_NUM_RXP + 1)) {
3737                 bnad_q_num_adjust(bnad, bna_attr(bna)->num_txq - 1,
3738                         bna_attr(bna)->num_rxp - 1);
3739                 if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) ||
3740                         bna_num_rxp_set(bna, BNAD_NUM_RXP + 1))
3741                         err = -EIO;
3742         }
3743         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3744         if (err)
3745                 goto disable_ioceth;
3746
3747         spin_lock_irqsave(&bnad->bna_lock, flags);
3748         bna_mod_res_req(&bnad->bna, &bnad->mod_res_info[0]);
3749         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3750
3751         err = bnad_res_alloc(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX);
3752         if (err) {
3753                 err = -EIO;
3754                 goto disable_ioceth;
3755         }
3756
3757         spin_lock_irqsave(&bnad->bna_lock, flags);
3758         bna_mod_init(&bnad->bna, &bnad->mod_res_info[0]);
3759         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3760
3761         /* Get the burnt-in mac */
3762         spin_lock_irqsave(&bnad->bna_lock, flags);
3763         bna_enet_perm_mac_get(&bna->enet, bnad->perm_addr);
3764         bnad_set_netdev_perm_addr(bnad);
3765         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3766
3767         mutex_unlock(&bnad->conf_mutex);
3768
3769         /* Finally, reguister with net_device layer */
3770         err = register_netdev(netdev);
3771         if (err) {
3772                 dev_err(&pdev->dev, "registering net device failed\n");
3773                 goto probe_uninit;
3774         }
3775         set_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags);
3776
3777         return 0;
3778
3779 probe_success:
3780         mutex_unlock(&bnad->conf_mutex);
3781         return 0;
3782
3783 probe_uninit:
3784         mutex_lock(&bnad->conf_mutex);
3785         bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX);
3786 disable_ioceth:
3787         bnad_ioceth_disable(bnad);
3788         del_timer_sync(&bnad->bna.ioceth.ioc.ioc_timer);
3789         del_timer_sync(&bnad->bna.ioceth.ioc.sem_timer);
3790         del_timer_sync(&bnad->bna.ioceth.ioc.hb_timer);
3791         spin_lock_irqsave(&bnad->bna_lock, flags);
3792         bna_uninit(bna);
3793         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3794         bnad_mbox_irq_free(bnad);
3795         bnad_disable_msix(bnad);
3796 res_free:
3797         bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX);
3798 drv_uninit:
3799         /* Remove the debugfs node for this bnad */
3800         kfree(bnad->regdata);
3801         bnad_debugfs_uninit(bnad);
3802         bnad_uninit(bnad);
3803 pci_uninit:
3804         bnad_pci_uninit(pdev);
3805 unlock_mutex:
3806         mutex_unlock(&bnad->conf_mutex);
3807         bnad_remove_from_list(bnad);
3808         bnad_lock_uninit(bnad);
3809         free_netdev(netdev);
3810         return err;
3811 }
3812
3813 static void
3814 bnad_pci_remove(struct pci_dev *pdev)
3815 {
3816         struct net_device *netdev = pci_get_drvdata(pdev);
3817         struct bnad *bnad;
3818         struct bna *bna;
3819         unsigned long flags;
3820
3821         if (!netdev)
3822                 return;
3823
3824         bnad = netdev_priv(netdev);
3825         bna = &bnad->bna;
3826
3827         if (test_and_clear_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags))
3828                 unregister_netdev(netdev);
3829
3830         mutex_lock(&bnad->conf_mutex);
3831         bnad_ioceth_disable(bnad);
3832         del_timer_sync(&bnad->bna.ioceth.ioc.ioc_timer);
3833         del_timer_sync(&bnad->bna.ioceth.ioc.sem_timer);
3834         del_timer_sync(&bnad->bna.ioceth.ioc.hb_timer);
3835         spin_lock_irqsave(&bnad->bna_lock, flags);
3836         bna_uninit(bna);
3837         spin_unlock_irqrestore(&bnad->bna_lock, flags);
3838
3839         bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX);
3840         bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX);
3841         bnad_mbox_irq_free(bnad);
3842         bnad_disable_msix(bnad);
3843         bnad_pci_uninit(pdev);
3844         mutex_unlock(&bnad->conf_mutex);
3845         bnad_remove_from_list(bnad);
3846         bnad_lock_uninit(bnad);
3847         /* Remove the debugfs node for this bnad */
3848         kfree(bnad->regdata);
3849         bnad_debugfs_uninit(bnad);
3850         bnad_uninit(bnad);
3851         free_netdev(netdev);
3852 }
3853
3854 static const struct pci_device_id bnad_pci_id_table[] = {
3855         {
3856                 PCI_DEVICE(PCI_VENDOR_ID_BROCADE,
3857                         PCI_DEVICE_ID_BROCADE_CT),
3858                 .class = PCI_CLASS_NETWORK_ETHERNET << 8,
3859                 .class_mask =  0xffff00
3860         },
3861         {
3862                 PCI_DEVICE(PCI_VENDOR_ID_BROCADE,
3863                         BFA_PCI_DEVICE_ID_CT2),
3864                 .class = PCI_CLASS_NETWORK_ETHERNET << 8,
3865                 .class_mask =  0xffff00
3866         },
3867         {0,  },
3868 };
3869
3870 MODULE_DEVICE_TABLE(pci, bnad_pci_id_table);
3871
3872 static struct pci_driver bnad_pci_driver = {
3873         .name = BNAD_NAME,
3874         .id_table = bnad_pci_id_table,
3875         .probe = bnad_pci_probe,
3876         .remove = bnad_pci_remove,
3877 };
3878
3879 static int __init
3880 bnad_module_init(void)
3881 {
3882         int err;
3883
3884         pr_info("bna: QLogic BR-series 10G Ethernet driver - version: %s\n",
3885                 BNAD_VERSION);
3886
3887         bfa_nw_ioc_auto_recover(bnad_ioc_auto_recover);
3888
3889         err = pci_register_driver(&bnad_pci_driver);
3890         if (err < 0) {
3891                 pr_err("bna: PCI driver registration failed err=%d\n", err);
3892                 return err;
3893         }
3894
3895         return 0;
3896 }
3897
3898 static void __exit
3899 bnad_module_exit(void)
3900 {
3901         pci_unregister_driver(&bnad_pci_driver);
3902         release_firmware(bfi_fw);
3903 }
3904
3905 module_init(bnad_module_init);
3906 module_exit(bnad_module_exit);
3907
3908 MODULE_AUTHOR("Brocade");
3909 MODULE_LICENSE("GPL");
3910 MODULE_DESCRIPTION("QLogic BR-series 10G PCIe Ethernet driver");
3911 MODULE_VERSION(BNAD_VERSION);
3912 MODULE_FIRMWARE(CNA_FW_FILE_CT);
3913 MODULE_FIRMWARE(CNA_FW_FILE_CT2);