drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9
  10 #include <linux/module.h>
  11
  12 #include <linux/stringify.h>
  13 #include <linux/kernel.h>
  14 #include <linux/timer.h>
  15 #include <linux/errno.h>
  16 #include <linux/ioport.h>
  17 #include <linux/slab.h>
  18 #include <linux/vmalloc.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/etherdevice.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/bitops.h>
  26 #include <linux/io.h>
  27 #include <linux/irq.h>
  28 #include <linux/delay.h>
  29 #include <asm/byteorder.h>
  30 #include <asm/page.h>
  31 #include <linux/time.h>
  32 #include <linux/mii.h>
  33 #include <linux/if.h>
  34 #include <linux/if_vlan.h>
  35 #include <net/ip.h>
  36 #include <net/tcp.h>
  37 #include <net/udp.h>
  38 #include <net/checksum.h>
  39 #include <net/ip6_checksum.h>
  40 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
  41 #include <net/vxlan.h>
  42 #endif
  43 #ifdef CONFIG_NET_RX_BUSY_POLL
  44 #include <net/busy_poll.h>
  45 #endif
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53
  54 #include "bnxt_hsi.h"
  55 #include "bnxt.h"
  56 #include "bnxt_sriov.h"
  57 #include "bnxt_ethtool.h"
  58
  59 #define BNXT_TX_TIMEOUT         (5 * HZ)
  60
  61 static const char version[] =
  62         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  63
  64 MODULE_LICENSE("GPL");
  65 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  66 MODULE_VERSION(DRV_MODULE_VERSION);
  67
  68 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  69 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  70 #define BNXT_RX_COPY_THRESH 256
  71
  72 #define BNXT_TX_PUSH_THRESH 164
  73
  74 enum board_idx {
  75         BCM57301,
  76         BCM57302,
  77         BCM57304,
  78         BCM57402,
  79         BCM57404,
  80         BCM57406,
  81         BCM57404_NPAR,
  82         BCM57314,
  83         BCM57304_VF,
  84         BCM57404_VF,
  85 };
  86
  87 /* indexed by enum above */
  88 static const struct {
  89         char *name;
  90 } board_info[] = {
  91         { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
  92         { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
  93         { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
  94         { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
  95         { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
  96         { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
  97         { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
  98         { "Broadcom BCM57314 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
  99         { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
 100         { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
 101 };
 102
 103 static const struct pci_device_id bnxt_pci_tbl[] = {
 104         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 105         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 106         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 107         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 108         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 109         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 110         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57404_NPAR },
 111         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 112 #ifdef CONFIG_BNXT_SRIOV
 113         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
 114         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
 115 #endif
 116         { 0 }
 117 };
 118
 119 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 120
 121 static const u16 bnxt_vf_req_snif[] = {
 122         HWRM_FUNC_CFG,
 123         HWRM_PORT_PHY_QCFG,
 124         HWRM_CFA_L2_FILTER_ALLOC,
 125 };
 126
 127 static const u16 bnxt_async_events_arr[] = {
 128         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 129         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 130         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 131         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 132         HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 133 };
 134
 135 static bool bnxt_vf_pciid(enum board_idx idx)
 136 {
 137         return (idx == BCM57304_VF || idx == BCM57404_VF);
 138 }
 139
 140 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 141 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 142 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 143
 144 #define BNXT_CP_DB_REARM(db, raw_cons)                                  \
 145                 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
 146
 147 #define BNXT_CP_DB(db, raw_cons)                                        \
 148                 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
 149
 150 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 151                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 152
 153 static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
 154 {
 155         /* Tell compiler to fetch tx indices from memory. */
 156         barrier();
 157
 158         return bp->tx_ring_size -
 159                 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
 160 }
 161
 162 static const u16 bnxt_lhint_arr[] = {
 163         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 164         TX_BD_FLAGS_LHINT_512_TO_1023,
 165         TX_BD_FLAGS_LHINT_1024_TO_2047,
 166         TX_BD_FLAGS_LHINT_1024_TO_2047,
 167         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 168         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 169         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 170         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 171         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 172         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 173         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 174         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 175         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 176         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 177         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 178         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 179         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 180         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 181         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 182 };
 183
 184 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 185 {
 186         struct bnxt *bp = netdev_priv(dev);
 187         struct tx_bd *txbd;
 188         struct tx_bd_ext *txbd1;
 189         struct netdev_queue *txq;
 190         int i;
 191         dma_addr_t mapping;
 192         unsigned int length, pad = 0;
 193         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 194         u16 prod, last_frag;
 195         struct pci_dev *pdev = bp->pdev;
 196         struct bnxt_tx_ring_info *txr;
 197         struct bnxt_sw_tx_bd *tx_buf;
 198
 199         i = skb_get_queue_mapping(skb);
 200         if (unlikely(i >= bp->tx_nr_rings)) {
 201                 dev_kfree_skb_any(skb);
 202                 return NETDEV_TX_OK;
 203         }
 204
 205         txr = &bp->tx_ring[i];
 206         txq = netdev_get_tx_queue(dev, i);
 207         prod = txr->tx_prod;
 208
 209         free_size = bnxt_tx_avail(bp, txr);
 210         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 211                 netif_tx_stop_queue(txq);
 212                 return NETDEV_TX_BUSY;
 213         }
 214
 215         length = skb->len;
 216         len = skb_headlen(skb);
 217         last_frag = skb_shinfo(skb)->nr_frags;
 218
 219         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 220
 221         txbd->tx_bd_opaque = prod;
 222
 223         tx_buf = &txr->tx_buf_ring[prod];
 224         tx_buf->skb = skb;
 225         tx_buf->nr_frags = last_frag;
 226
 227         vlan_tag_flags = 0;
 228         cfa_action = 0;
 229         if (skb_vlan_tag_present(skb)) {
 230                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 231                                  skb_vlan_tag_get(skb);
 232                 /* Currently supports 8021Q, 8021AD vlan offloads
 233                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 234                  */
 235                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 236                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 237         }
 238
 239         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 240                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 241                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 242                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 243                 void *pdata = tx_push_buf->data;
 244                 u64 *end;
 245                 int j, push_len;
 246
 247                 /* Set COAL_NOW to be ready quickly for the next push */
 248                 tx_push->tx_bd_len_flags_type =
 249                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 250                                         TX_BD_TYPE_LONG_TX_BD |
 251                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 252                                         TX_BD_FLAGS_COAL_NOW |
 253                                         TX_BD_FLAGS_PACKET_END |
 254                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 255
 256                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 257                         tx_push1->tx_bd_hsize_lflags =
 258                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 259                 else
 260                         tx_push1->tx_bd_hsize_lflags = 0;
 261
 262                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 263                 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
 264
 265                 end = pdata + length;
 266                 end = PTR_ALIGN(end, 8) - 1;
 267                 *end = 0;
 268
 269                 skb_copy_from_linear_data(skb, pdata, len);
 270                 pdata += len;
 271                 for (j = 0; j < last_frag; j++) {
 272                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 273                         void *fptr;
 274
 275                         fptr = skb_frag_address_safe(frag);
 276                         if (!fptr)
 277                                 goto normal_tx;
 278
 279                         memcpy(pdata, fptr, skb_frag_size(frag));
 280                         pdata += skb_frag_size(frag);
 281                 }
 282
 283                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 284                 txbd->tx_bd_haddr = txr->data_mapping;
 285                 prod = NEXT_TX(prod);
 286                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 287                 memcpy(txbd, tx_push1, sizeof(*txbd));
 288                 prod = NEXT_TX(prod);
 289                 tx_push->doorbell =
 290                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 291                 txr->tx_prod = prod;
 292
 293                 tx_buf->is_push = 1;
 294                 netdev_tx_sent_queue(txq, skb->len);
 295                 wmb();  /* Sync is_push and byte queue before pushing data */
 296
 297                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 298                 if (push_len > 16) {
 299                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
 300                         __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
 301                                          push_len - 16);
 302                 } else {
 303                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 304                                          push_len);
 305                 }
 306
 307                 goto tx_done;
 308         }
 309
 310 normal_tx:
 311         if (length < BNXT_MIN_PKT_SIZE) {
 312                 pad = BNXT_MIN_PKT_SIZE - length;
 313                 if (skb_pad(skb, pad)) {
 314                         /* SKB already freed. */
 315                         tx_buf->skb = NULL;
 316                         return NETDEV_TX_OK;
 317                 }
 318                 length = BNXT_MIN_PKT_SIZE;
 319         }
 320
 321         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 322
 323         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 324                 dev_kfree_skb_any(skb);
 325                 tx_buf->skb = NULL;
 326                 return NETDEV_TX_OK;
 327         }
 328
 329         dma_unmap_addr_set(tx_buf, mapping, mapping);
 330         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 331                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 332
 333         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 334
 335         prod = NEXT_TX(prod);
 336         txbd1 = (struct tx_bd_ext *)
 337                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 338
 339         txbd1->tx_bd_hsize_lflags = 0;
 340         if (skb_is_gso(skb)) {
 341                 u32 hdr_len;
 342
 343                 if (skb->encapsulation)
 344                         hdr_len = skb_inner_network_offset(skb) +
 345                                 skb_inner_network_header_len(skb) +
 346                                 inner_tcp_hdrlen(skb);
 347                 else
 348                         hdr_len = skb_transport_offset(skb) +
 349                                 tcp_hdrlen(skb);
 350
 351                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 352                                         TX_BD_FLAGS_T_IPID |
 353                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 354                 length = skb_shinfo(skb)->gso_size;
 355                 txbd1->tx_bd_mss = cpu_to_le32(length);
 356                 length += hdr_len;
 357         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 358                 txbd1->tx_bd_hsize_lflags =
 359                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 360                 txbd1->tx_bd_mss = 0;
 361         }
 362
 363         length >>= 9;
 364         flags |= bnxt_lhint_arr[length];
 365         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 366
 367         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 368         txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
 369         for (i = 0; i < last_frag; i++) {
 370                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 371
 372                 prod = NEXT_TX(prod);
 373                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 374
 375                 len = skb_frag_size(frag);
 376                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 377                                            DMA_TO_DEVICE);
 378
 379                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 380                         goto tx_dma_error;
 381
 382                 tx_buf = &txr->tx_buf_ring[prod];
 383                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 384
 385                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 386
 387                 flags = len << TX_BD_LEN_SHIFT;
 388                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 389         }
 390
 391         flags &= ~TX_BD_LEN;
 392         txbd->tx_bd_len_flags_type =
 393                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 394                             TX_BD_FLAGS_PACKET_END);
 395
 396         netdev_tx_sent_queue(txq, skb->len);
 397
 398         /* Sync BD data before updating doorbell */
 399         wmb();
 400
 401         prod = NEXT_TX(prod);
 402         txr->tx_prod = prod;
 403
 404         writel(DB_KEY_TX | prod, txr->tx_doorbell);
 405         writel(DB_KEY_TX | prod, txr->tx_doorbell);
 406
 407 tx_done:
 408
 409         mmiowb();
 410
 411         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 412                 netif_tx_stop_queue(txq);
 413
 414                 /* netif_tx_stop_queue() must be done before checking
 415                  * tx index in bnxt_tx_avail() below, because in
 416                  * bnxt_tx_int(), we update tx index before checking for
 417                  * netif_tx_queue_stopped().
 418                  */
 419                 smp_mb();
 420                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 421                         netif_tx_wake_queue(txq);
 422         }
 423         return NETDEV_TX_OK;
 424
 425 tx_dma_error:
 426         last_frag = i;
 427
 428         /* start back at beginning and unmap skb */
 429         prod = txr->tx_prod;
 430         tx_buf = &txr->tx_buf_ring[prod];
 431         tx_buf->skb = NULL;
 432         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 433                          skb_headlen(skb), PCI_DMA_TODEVICE);
 434         prod = NEXT_TX(prod);
 435
 436         /* unmap remaining mapped pages */
 437         for (i = 0; i < last_frag; i++) {
 438                 prod = NEXT_TX(prod);
 439                 tx_buf = &txr->tx_buf_ring[prod];
 440                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 441                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 442                                PCI_DMA_TODEVICE);
 443         }
 444
 445         dev_kfree_skb_any(skb);
 446         return NETDEV_TX_OK;
 447 }
 448
 449 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 450 {
 451         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 452         int index = txr - &bp->tx_ring[0];
 453         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index);
 454         u16 cons = txr->tx_cons;
 455         struct pci_dev *pdev = bp->pdev;
 456         int i;
 457         unsigned int tx_bytes = 0;
 458
 459         for (i = 0; i < nr_pkts; i++) {
 460                 struct bnxt_sw_tx_bd *tx_buf;
 461                 struct sk_buff *skb;
 462                 int j, last;
 463
 464                 tx_buf = &txr->tx_buf_ring[cons];
 465                 cons = NEXT_TX(cons);
 466                 skb = tx_buf->skb;
 467                 tx_buf->skb = NULL;
 468
 469                 if (tx_buf->is_push) {
 470                         tx_buf->is_push = 0;
 471                         goto next_tx_int;
 472                 }
 473
 474                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 475                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 476                 last = tx_buf->nr_frags;
 477
 478                 for (j = 0; j < last; j++) {
 479                         cons = NEXT_TX(cons);
 480                         tx_buf = &txr->tx_buf_ring[cons];
 481                         dma_unmap_page(
 482                                 &pdev->dev,
 483                                 dma_unmap_addr(tx_buf, mapping),
 484                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 485                                 PCI_DMA_TODEVICE);
 486                 }
 487
 488 next_tx_int:
 489                 cons = NEXT_TX(cons);
 490
 491                 tx_bytes += skb->len;
 492                 dev_kfree_skb_any(skb);
 493         }
 494
 495         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 496         txr->tx_cons = cons;
 497
 498         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 499          * before checking for netif_tx_queue_stopped().  Without the
 500          * memory barrier, there is a small possibility that bnxt_start_xmit()
 501          * will miss it and cause the queue to be stopped forever.
 502          */
 503         smp_mb();
 504
 505         if (unlikely(netif_tx_queue_stopped(txq)) &&
 506             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 507                 __netif_tx_lock(txq, smp_processor_id());
 508                 if (netif_tx_queue_stopped(txq) &&
 509                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 510                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 511                         netif_tx_wake_queue(txq);
 512                 __netif_tx_unlock(txq);
 513         }
 514 }
 515
 516 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 517                                        gfp_t gfp)
 518 {
 519         u8 *data;
 520         struct pci_dev *pdev = bp->pdev;
 521
 522         data = kmalloc(bp->rx_buf_size, gfp);
 523         if (!data)
 524                 return NULL;
 525
 526         *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET,
 527                                   bp->rx_buf_use_size, PCI_DMA_FROMDEVICE);
 528
 529         if (dma_mapping_error(&pdev->dev, *mapping)) {
 530                 kfree(data);
 531                 data = NULL;
 532         }
 533         return data;
 534 }
 535
 536 static inline int bnxt_alloc_rx_data(struct bnxt *bp,
 537                                      struct bnxt_rx_ring_info *rxr,
 538                                      u16 prod, gfp_t gfp)
 539 {
 540         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 541         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 542         u8 *data;
 543         dma_addr_t mapping;
 544
 545         data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 546         if (!data)
 547                 return -ENOMEM;
 548
 549         rx_buf->data = data;
 550         dma_unmap_addr_set(rx_buf, mapping, mapping);
 551
 552         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 553
 554         return 0;
 555 }
 556
 557 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,
 558                                u8 *data)
 559 {
 560         u16 prod = rxr->rx_prod;
 561         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 562         struct rx_bd *cons_bd, *prod_bd;
 563
 564         prod_rx_buf = &rxr->rx_buf_ring[prod];
 565         cons_rx_buf = &rxr->rx_buf_ring[cons];
 566
 567         prod_rx_buf->data = data;
 568
 569         dma_unmap_addr_set(prod_rx_buf, mapping,
 570                            dma_unmap_addr(cons_rx_buf, mapping));
 571
 572         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 573         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 574
 575         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 576 }
 577
 578 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 579 {
 580         u16 next, max = rxr->rx_agg_bmap_size;
 581
 582         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 583         if (next >= max)
 584                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 585         return next;
 586 }
 587
 588 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 589                                      struct bnxt_rx_ring_info *rxr,
 590                                      u16 prod, gfp_t gfp)
 591 {
 592         struct rx_bd *rxbd =
 593                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 594         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 595         struct pci_dev *pdev = bp->pdev;
 596         struct page *page;
 597         dma_addr_t mapping;
 598         u16 sw_prod = rxr->rx_sw_agg_prod;
 599         unsigned int offset = 0;
 600
 601         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 602                 page = rxr->rx_page;
 603                 if (!page) {
 604                         page = alloc_page(gfp);
 605                         if (!page)
 606                                 return -ENOMEM;
 607                         rxr->rx_page = page;
 608                         rxr->rx_page_offset = 0;
 609                 }
 610                 offset = rxr->rx_page_offset;
 611                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 612                 if (rxr->rx_page_offset == PAGE_SIZE)
 613                         rxr->rx_page = NULL;
 614                 else
 615                         get_page(page);
 616         } else {
 617                 page = alloc_page(gfp);
 618                 if (!page)
 619                         return -ENOMEM;
 620         }
 621
 622         mapping = dma_map_page(&pdev->dev, page, offset, BNXT_RX_PAGE_SIZE,
 623                                PCI_DMA_FROMDEVICE);
 624         if (dma_mapping_error(&pdev->dev, mapping)) {
 625                 __free_page(page);
 626                 return -EIO;
 627         }
 628
 629         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 630                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 631
 632         __set_bit(sw_prod, rxr->rx_agg_bmap);
 633         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 634         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 635
 636         rx_agg_buf->page = page;
 637         rx_agg_buf->offset = offset;
 638         rx_agg_buf->mapping = mapping;
 639         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 640         rxbd->rx_bd_opaque = sw_prod;
 641         return 0;
 642 }
 643
 644 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
 645                                    u32 agg_bufs)
 646 {
 647         struct bnxt *bp = bnapi->bp;
 648         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 649         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 650         u16 prod = rxr->rx_agg_prod;
 651         u16 sw_prod = rxr->rx_sw_agg_prod;
 652         u32 i;
 653
 654         for (i = 0; i < agg_bufs; i++) {
 655                 u16 cons;
 656                 struct rx_agg_cmp *agg;
 657                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 658                 struct rx_bd *prod_bd;
 659                 struct page *page;
 660
 661                 agg = (struct rx_agg_cmp *)
 662                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 663                 cons = agg->rx_agg_cmp_opaque;
 664                 __clear_bit(cons, rxr->rx_agg_bmap);
 665
 666                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 667                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 668
 669                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 670                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 671                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 672
 673                 /* It is possible for sw_prod to be equal to cons, so
 674                  * set cons_rx_buf->page to NULL first.
 675                  */
 676                 page = cons_rx_buf->page;
 677                 cons_rx_buf->page = NULL;
 678                 prod_rx_buf->page = page;
 679                 prod_rx_buf->offset = cons_rx_buf->offset;
 680
 681                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 682
 683                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 684
 685                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 686                 prod_bd->rx_bd_opaque = sw_prod;
 687
 688                 prod = NEXT_RX_AGG(prod);
 689                 sw_prod = NEXT_RX_AGG(sw_prod);
 690                 cp_cons = NEXT_CMP(cp_cons);
 691         }
 692         rxr->rx_agg_prod = prod;
 693         rxr->rx_sw_agg_prod = sw_prod;
 694 }
 695
 696 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 697                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 698                                    u16 prod, u8 *data, dma_addr_t dma_addr,
 699                                    unsigned int len)
 700 {
 701         int err;
 702         struct sk_buff *skb;
 703
 704         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 705         if (unlikely(err)) {
 706                 bnxt_reuse_rx_data(rxr, cons, data);
 707                 return NULL;
 708         }
 709
 710         skb = build_skb(data, 0);
 711         dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 712                          PCI_DMA_FROMDEVICE);
 713         if (!skb) {
 714                 kfree(data);
 715                 return NULL;
 716         }
 717
 718         skb_reserve(skb, BNXT_RX_OFFSET);
 719         skb_put(skb, len);
 720         return skb;
 721 }
 722
 723 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
 724                                      struct sk_buff *skb, u16 cp_cons,
 725                                      u32 agg_bufs)
 726 {
 727         struct pci_dev *pdev = bp->pdev;
 728         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 729         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 730         u16 prod = rxr->rx_agg_prod;
 731         u32 i;
 732
 733         for (i = 0; i < agg_bufs; i++) {
 734                 u16 cons, frag_len;
 735                 struct rx_agg_cmp *agg;
 736                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 737                 struct page *page;
 738                 dma_addr_t mapping;
 739
 740                 agg = (struct rx_agg_cmp *)
 741                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 742                 cons = agg->rx_agg_cmp_opaque;
 743                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 744                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 745
 746                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 747                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 748                                    cons_rx_buf->offset, frag_len);
 749                 __clear_bit(cons, rxr->rx_agg_bmap);
 750
 751                 /* It is possible for bnxt_alloc_rx_page() to allocate
 752                  * a sw_prod index that equals the cons index, so we
 753                  * need to clear the cons entry now.
 754                  */
 755                 mapping = dma_unmap_addr(cons_rx_buf, mapping);
 756                 page = cons_rx_buf->page;
 757                 cons_rx_buf->page = NULL;
 758
 759                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 760                         struct skb_shared_info *shinfo;
 761                         unsigned int nr_frags;
 762
 763                         shinfo = skb_shinfo(skb);
 764                         nr_frags = --shinfo->nr_frags;
 765                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 766
 767                         dev_kfree_skb(skb);
 768
 769                         cons_rx_buf->page = page;
 770
 771                         /* Update prod since possibly some pages have been
 772                          * allocated already.
 773                          */
 774                         rxr->rx_agg_prod = prod;
 775                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
 776                         return NULL;
 777                 }
 778
 779                 dma_unmap_page(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
 780                                PCI_DMA_FROMDEVICE);
 781
 782                 skb->data_len += frag_len;
 783                 skb->len += frag_len;
 784                 skb->truesize += PAGE_SIZE;
 785
 786                 prod = NEXT_RX_AGG(prod);
 787                 cp_cons = NEXT_CMP(cp_cons);
 788         }
 789         rxr->rx_agg_prod = prod;
 790         return skb;
 791 }
 792
 793 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 794                                u8 agg_bufs, u32 *raw_cons)
 795 {
 796         u16 last;
 797         struct rx_agg_cmp *agg;
 798
 799         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
 800         last = RING_CMP(*raw_cons);
 801         agg = (struct rx_agg_cmp *)
 802                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
 803         return RX_AGG_CMP_VALID(agg, *raw_cons);
 804 }
 805
 806 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
 807                                             unsigned int len,
 808                                             dma_addr_t mapping)
 809 {
 810         struct bnxt *bp = bnapi->bp;
 811         struct pci_dev *pdev = bp->pdev;
 812         struct sk_buff *skb;
 813
 814         skb = napi_alloc_skb(&bnapi->napi, len);
 815         if (!skb)
 816                 return NULL;
 817
 818         dma_sync_single_for_cpu(&pdev->dev, mapping,
 819                                 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE);
 820
 821         memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET);
 822
 823         dma_sync_single_for_device(&pdev->dev, mapping,
 824                                    bp->rx_copy_thresh,
 825                                    PCI_DMA_FROMDEVICE);
 826
 827         skb_put(skb, len);
 828         return skb;
 829 }
 830
 831 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
 832                            u32 *raw_cons, void *cmp)
 833 {
 834         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 835         struct rx_cmp *rxcmp = cmp;
 836         u32 tmp_raw_cons = *raw_cons;
 837         u8 cmp_type, agg_bufs = 0;
 838
 839         cmp_type = RX_CMP_TYPE(rxcmp);
 840
 841         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
 842                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
 843                             RX_CMP_AGG_BUFS) >>
 844                            RX_CMP_AGG_BUFS_SHIFT;
 845         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
 846                 struct rx_tpa_end_cmp *tpa_end = cmp;
 847
 848                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
 849                             RX_TPA_END_CMP_AGG_BUFS) >>
 850                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
 851         }
 852
 853         if (agg_bufs) {
 854                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
 855                         return -EBUSY;
 856         }
 857         *raw_cons = tmp_raw_cons;
 858         return 0;
 859 }
 860
 861 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 862 {
 863         if (!rxr->bnapi->in_reset) {
 864                 rxr->bnapi->in_reset = true;
 865                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
 866                 schedule_work(&bp->sp_task);
 867         }
 868         rxr->rx_next_cons = 0xffff;
 869 }
 870
 871 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 872                            struct rx_tpa_start_cmp *tpa_start,
 873                            struct rx_tpa_start_cmp_ext *tpa_start1)
 874 {
 875         u8 agg_id = TPA_START_AGG_ID(tpa_start);
 876         u16 cons, prod;
 877         struct bnxt_tpa_info *tpa_info;
 878         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 879         struct rx_bd *prod_bd;
 880         dma_addr_t mapping;
 881
 882         cons = tpa_start->rx_tpa_start_cmp_opaque;
 883         prod = rxr->rx_prod;
 884         cons_rx_buf = &rxr->rx_buf_ring[cons];
 885         prod_rx_buf = &rxr->rx_buf_ring[prod];
 886         tpa_info = &rxr->rx_tpa[agg_id];
 887
 888         if (unlikely(cons != rxr->rx_next_cons)) {
 889                 bnxt_sched_reset(bp, rxr);
 890                 return;
 891         }
 892
 893         prod_rx_buf->data = tpa_info->data;
 894
 895         mapping = tpa_info->mapping;
 896         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 897
 898         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 899
 900         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
 901
 902         tpa_info->data = cons_rx_buf->data;
 903         cons_rx_buf->data = NULL;
 904         tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping);
 905
 906         tpa_info->len =
 907                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
 908                                 RX_TPA_START_CMP_LEN_SHIFT;
 909         if (likely(TPA_START_HASH_VALID(tpa_start))) {
 910                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
 911
 912                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
 913                 tpa_info->gso_type = SKB_GSO_TCPV4;
 914                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
 915                 if (hash_type == 3)
 916                         tpa_info->gso_type = SKB_GSO_TCPV6;
 917                 tpa_info->rss_hash =
 918                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
 919         } else {
 920                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
 921                 tpa_info->gso_type = 0;
 922                 if (netif_msg_rx_err(bp))
 923                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
 924         }
 925         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
 926         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
 927
 928         rxr->rx_prod = NEXT_RX(prod);
 929         cons = NEXT_RX(cons);
 930         rxr->rx_next_cons = NEXT_RX(cons);
 931         cons_rx_buf = &rxr->rx_buf_ring[cons];
 932
 933         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
 934         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
 935         cons_rx_buf->data = NULL;
 936 }
 937
 938 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
 939                            u16 cp_cons, u32 agg_bufs)
 940 {
 941         if (agg_bufs)
 942                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
 943 }
 944
 945 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
 946 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
 947
 948 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
 949                                            int payload_off, int tcp_ts,
 950                                            struct sk_buff *skb)
 951 {
 952 #ifdef CONFIG_INET
 953         struct tcphdr *th;
 954         int len, nw_off, tcp_opt_len;
 955
 956         if (tcp_ts)
 957                 tcp_opt_len = 12;
 958
 959         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
 960                 struct iphdr *iph;
 961
 962                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
 963                          ETH_HLEN;
 964                 skb_set_network_header(skb, nw_off);
 965                 iph = ip_hdr(skb);
 966                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
 967                 len = skb->len - skb_transport_offset(skb);
 968                 th = tcp_hdr(skb);
 969                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
 970         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
 971                 struct ipv6hdr *iph;
 972
 973                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
 974                          ETH_HLEN;
 975                 skb_set_network_header(skb, nw_off);
 976                 iph = ipv6_hdr(skb);
 977                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
 978                 len = skb->len - skb_transport_offset(skb);
 979                 th = tcp_hdr(skb);
 980                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
 981         } else {
 982                 dev_kfree_skb_any(skb);
 983                 return NULL;
 984         }
 985         tcp_gro_complete(skb);
 986
 987         if (nw_off) { /* tunnel */
 988                 struct udphdr *uh = NULL;
 989
 990                 if (skb->protocol == htons(ETH_P_IP)) {
 991                         struct iphdr *iph = (struct iphdr *)skb->data;
 992
 993                         if (iph->protocol == IPPROTO_UDP)
 994                                 uh = (struct udphdr *)(iph + 1);
 995                 } else {
 996                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
 997
 998                         if (iph->nexthdr == IPPROTO_UDP)
 999                                 uh = (struct udphdr *)(iph + 1);
1000                 }
1001                 if (uh) {
1002                         if (uh->check)
1003                                 skb_shinfo(skb)->gso_type |=
1004                                         SKB_GSO_UDP_TUNNEL_CSUM;
1005                         else
1006                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1007                 }
1008         }
1009 #endif
1010         return skb;
1011 }
1012
1013 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1014                                            struct bnxt_tpa_info *tpa_info,
1015                                            struct rx_tpa_end_cmp *tpa_end,
1016                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1017                                            struct sk_buff *skb)
1018 {
1019 #ifdef CONFIG_INET
1020         int payload_off;
1021         u16 segs;
1022
1023         segs = TPA_END_TPA_SEGS(tpa_end);
1024         if (segs == 1)
1025                 return skb;
1026
1027         NAPI_GRO_CB(skb)->count = segs;
1028         skb_shinfo(skb)->gso_size =
1029                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1030         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1031         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1032                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1033                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1034         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1035 #endif
1036         return skb;
1037 }
1038
1039 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1040                                            struct bnxt_napi *bnapi,
1041                                            u32 *raw_cons,
1042                                            struct rx_tpa_end_cmp *tpa_end,
1043                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1044                                            bool *agg_event)
1045 {
1046         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1047         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1048         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1049         u8 *data, agg_bufs;
1050         u16 cp_cons = RING_CMP(*raw_cons);
1051         unsigned int len;
1052         struct bnxt_tpa_info *tpa_info;
1053         dma_addr_t mapping;
1054         struct sk_buff *skb;
1055
1056         if (unlikely(bnapi->in_reset)) {
1057                 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1058
1059                 if (rc < 0)
1060                         return ERR_PTR(-EBUSY);
1061                 return NULL;
1062         }
1063
1064         tpa_info = &rxr->rx_tpa[agg_id];
1065         data = tpa_info->data;
1066         prefetch(data);
1067         len = tpa_info->len;
1068         mapping = tpa_info->mapping;
1069
1070         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1071                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1072
1073         if (agg_bufs) {
1074                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1075                         return ERR_PTR(-EBUSY);
1076
1077                 *agg_event = true;
1078                 cp_cons = NEXT_CMP(cp_cons);
1079         }
1080
1081         if (unlikely(agg_bufs > MAX_SKB_FRAGS)) {
1082                 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1083                 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1084                             agg_bufs, (int)MAX_SKB_FRAGS);
1085                 return NULL;
1086         }
1087
1088         if (len <= bp->rx_copy_thresh) {
1089                 skb = bnxt_copy_skb(bnapi, data, len, mapping);
1090                 if (!skb) {
1091                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1092                         return NULL;
1093                 }
1094         } else {
1095                 u8 *new_data;
1096                 dma_addr_t new_mapping;
1097
1098                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1099                 if (!new_data) {
1100                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1101                         return NULL;
1102                 }
1103
1104                 tpa_info->data = new_data;
1105                 tpa_info->mapping = new_mapping;
1106
1107                 skb = build_skb(data, 0);
1108                 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size,
1109                                  PCI_DMA_FROMDEVICE);
1110
1111                 if (!skb) {
1112                         kfree(data);
1113                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1114                         return NULL;
1115                 }
1116                 skb_reserve(skb, BNXT_RX_OFFSET);
1117                 skb_put(skb, len);
1118         }
1119
1120         if (agg_bufs) {
1121                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1122                 if (!skb) {
1123                         /* Page reuse already handled by bnxt_rx_pages(). */
1124                         return NULL;
1125                 }
1126         }
1127         skb->protocol = eth_type_trans(skb, bp->dev);
1128
1129         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1130                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1131
1132         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1133             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1134                 u16 vlan_proto = tpa_info->metadata >>
1135                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1136                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
1137
1138                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1139         }
1140
1141         skb_checksum_none_assert(skb);
1142         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1143                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1144                 skb->csum_level =
1145                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1146         }
1147
1148         if (TPA_END_GRO(tpa_end))
1149                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1150
1151         return skb;
1152 }
1153
1154 /* returns the following:
1155  * 1       - 1 packet successfully received
1156  * 0       - successful TPA_START, packet not completed yet
1157  * -EBUSY  - completion ring does not have all the agg buffers yet
1158  * -ENOMEM - packet aborted due to out of memory
1159  * -EIO    - packet aborted due to hw error indicated in BD
1160  */
1161 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1162                        bool *agg_event)
1163 {
1164         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1165         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1166         struct net_device *dev = bp->dev;
1167         struct rx_cmp *rxcmp;
1168         struct rx_cmp_ext *rxcmp1;
1169         u32 tmp_raw_cons = *raw_cons;
1170         u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1171         struct bnxt_sw_rx_bd *rx_buf;
1172         unsigned int len;
1173         u8 *data, agg_bufs, cmp_type;
1174         dma_addr_t dma_addr;
1175         struct sk_buff *skb;
1176         int rc = 0;
1177
1178         rxcmp = (struct rx_cmp *)
1179                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1180
1181         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1182         cp_cons = RING_CMP(tmp_raw_cons);
1183         rxcmp1 = (struct rx_cmp_ext *)
1184                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1185
1186         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1187                 return -EBUSY;
1188
1189         cmp_type = RX_CMP_TYPE(rxcmp);
1190
1191         prod = rxr->rx_prod;
1192
1193         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1194                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1195                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1196
1197                 goto next_rx_no_prod;
1198
1199         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1200                 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1201                                    (struct rx_tpa_end_cmp *)rxcmp,
1202                                    (struct rx_tpa_end_cmp_ext *)rxcmp1,
1203                                    agg_event);
1204
1205                 if (unlikely(IS_ERR(skb)))
1206                         return -EBUSY;
1207
1208                 rc = -ENOMEM;
1209                 if (likely(skb)) {
1210                         skb_record_rx_queue(skb, bnapi->index);
1211                         skb_mark_napi_id(skb, &bnapi->napi);
1212                         if (bnxt_busy_polling(bnapi))
1213                                 netif_receive_skb(skb);
1214                         else
1215                                 napi_gro_receive(&bnapi->napi, skb);
1216                         rc = 1;
1217                 }
1218                 goto next_rx_no_prod;
1219         }
1220
1221         cons = rxcmp->rx_cmp_opaque;
1222         rx_buf = &rxr->rx_buf_ring[cons];
1223         data = rx_buf->data;
1224         if (unlikely(cons != rxr->rx_next_cons)) {
1225                 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1226
1227                 bnxt_sched_reset(bp, rxr);
1228                 return rc1;
1229         }
1230         prefetch(data);
1231
1232         agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >>
1233                                 RX_CMP_AGG_BUFS_SHIFT;
1234
1235         if (agg_bufs) {
1236                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1237                         return -EBUSY;
1238
1239                 cp_cons = NEXT_CMP(cp_cons);
1240                 *agg_event = true;
1241         }
1242
1243         rx_buf->data = NULL;
1244         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1245                 bnxt_reuse_rx_data(rxr, cons, data);
1246                 if (agg_bufs)
1247                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1248
1249                 rc = -EIO;
1250                 goto next_rx;
1251         }
1252
1253         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1254         dma_addr = dma_unmap_addr(rx_buf, mapping);
1255
1256         if (len <= bp->rx_copy_thresh) {
1257                 skb = bnxt_copy_skb(bnapi, data, len, dma_addr);
1258                 bnxt_reuse_rx_data(rxr, cons, data);
1259                 if (!skb) {
1260                         rc = -ENOMEM;
1261                         goto next_rx;
1262                 }
1263         } else {
1264                 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);
1265                 if (!skb) {
1266                         rc = -ENOMEM;
1267                         goto next_rx;
1268                 }
1269         }
1270
1271         if (agg_bufs) {
1272                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1273                 if (!skb) {
1274                         rc = -ENOMEM;
1275                         goto next_rx;
1276                 }
1277         }
1278
1279         if (RX_CMP_HASH_VALID(rxcmp)) {
1280                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1281                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1282
1283                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1284                 if (hash_type != 1 && hash_type != 3)
1285                         type = PKT_HASH_TYPE_L3;
1286                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1287         }
1288
1289         skb->protocol = eth_type_trans(skb, dev);
1290
1291         if ((rxcmp1->rx_cmp_flags2 &
1292              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1293             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1294                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1295                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
1296                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1297
1298                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1299         }
1300
1301         skb_checksum_none_assert(skb);
1302         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1303                 if (dev->features & NETIF_F_RXCSUM) {
1304                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1305                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1306                 }
1307         } else {
1308                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1309                         if (dev->features & NETIF_F_RXCSUM)
1310                                 cpr->rx_l4_csum_errors++;
1311                 }
1312         }
1313
1314         skb_record_rx_queue(skb, bnapi->index);
1315         skb_mark_napi_id(skb, &bnapi->napi);
1316         if (bnxt_busy_polling(bnapi))
1317                 netif_receive_skb(skb);
1318         else
1319                 napi_gro_receive(&bnapi->napi, skb);
1320         rc = 1;
1321
1322 next_rx:
1323         rxr->rx_prod = NEXT_RX(prod);
1324         rxr->rx_next_cons = NEXT_RX(cons);
1325
1326 next_rx_no_prod:
1327         *raw_cons = tmp_raw_cons;
1328
1329         return rc;
1330 }
1331
1332 #define BNXT_GET_EVENT_PORT(data)       \
1333         ((data) &                               \
1334          HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1335
1336 static int bnxt_async_event_process(struct bnxt *bp,
1337                                     struct hwrm_async_event_cmpl *cmpl)
1338 {
1339         u16 event_id = le16_to_cpu(cmpl->event_id);
1340
1341         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1342         switch (event_id) {
1343         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1344                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1345                 struct bnxt_link_info *link_info = &bp->link_info;
1346
1347                 if (BNXT_VF(bp))
1348                         goto async_event_process_exit;
1349                 if (data1 & 0x20000) {
1350                         u16 fw_speed = link_info->force_link_speed;
1351                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1352
1353                         netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1354                                     speed);
1355                 }
1356                 /* fall thru */
1357         }
1358         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1359                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1360                 break;
1361         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1362                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1363                 break;
1364         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1365                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1366                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1367
1368                 if (BNXT_VF(bp))
1369                         break;
1370
1371                 if (bp->pf.port_id != port_id)
1372                         break;
1373
1374                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1375                 break;
1376         }
1377         case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1378                 if (BNXT_PF(bp))
1379                         goto async_event_process_exit;
1380                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1381                 break;
1382         default:
1383                 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
1384                            event_id);
1385                 goto async_event_process_exit;
1386         }
1387         schedule_work(&bp->sp_task);
1388 async_event_process_exit:
1389         return 0;
1390 }
1391
1392 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1393 {
1394         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1395         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1396         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1397                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1398
1399         switch (cmpl_type) {
1400         case CMPL_BASE_TYPE_HWRM_DONE:
1401                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1402                 if (seq_id == bp->hwrm_intr_seq_id)
1403                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1404                 else
1405                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1406                 break;
1407
1408         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1409                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1410
1411                 if ((vf_id < bp->pf.first_vf_id) ||
1412                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1413                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1414                                    vf_id);
1415                         return -EINVAL;
1416                 }
1417
1418                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1419                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1420                 schedule_work(&bp->sp_task);
1421                 break;
1422
1423         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1424                 bnxt_async_event_process(bp,
1425                                          (struct hwrm_async_event_cmpl *)txcmp);
1426
1427         default:
1428                 break;
1429         }
1430
1431         return 0;
1432 }
1433
1434 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1435 {
1436         struct bnxt_napi *bnapi = dev_instance;
1437         struct bnxt *bp = bnapi->bp;
1438         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1439         u32 cons = RING_CMP(cpr->cp_raw_cons);
1440
1441         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1442         napi_schedule(&bnapi->napi);
1443         return IRQ_HANDLED;
1444 }
1445
1446 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1447 {
1448         u32 raw_cons = cpr->cp_raw_cons;
1449         u16 cons = RING_CMP(raw_cons);
1450         struct tx_cmp *txcmp;
1451
1452         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1453
1454         return TX_CMP_VALID(txcmp, raw_cons);
1455 }
1456
1457 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1458 {
1459         struct bnxt_napi *bnapi = dev_instance;
1460         struct bnxt *bp = bnapi->bp;
1461         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1462         u32 cons = RING_CMP(cpr->cp_raw_cons);
1463         u32 int_status;
1464
1465         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1466
1467         if (!bnxt_has_work(bp, cpr)) {
1468                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1469                 /* return if erroneous interrupt */
1470                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1471                         return IRQ_NONE;
1472         }
1473
1474         /* disable ring IRQ */
1475         BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1476
1477         /* Return here if interrupt is shared and is disabled. */
1478         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1479                 return IRQ_HANDLED;
1480
1481         napi_schedule(&bnapi->napi);
1482         return IRQ_HANDLED;
1483 }
1484
1485 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1486 {
1487         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1488         u32 raw_cons = cpr->cp_raw_cons;
1489         u32 cons;
1490         int tx_pkts = 0;
1491         int rx_pkts = 0;
1492         bool rx_event = false;
1493         bool agg_event = false;
1494         struct tx_cmp *txcmp;
1495
1496         while (1) {
1497                 int rc;
1498
1499                 cons = RING_CMP(raw_cons);
1500                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1501
1502                 if (!TX_CMP_VALID(txcmp, raw_cons))
1503                         break;
1504
1505                 /* The valid test of the entry must be done first before
1506                  * reading any further.
1507                  */
1508                 dma_rmb();
1509                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1510                         tx_pkts++;
1511                         /* return full budget so NAPI will complete. */
1512                         if (unlikely(tx_pkts > bp->tx_wake_thresh))
1513                                 rx_pkts = budget;
1514                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1515                         rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event);
1516                         if (likely(rc >= 0))
1517                                 rx_pkts += rc;
1518                         else if (rc == -EBUSY)  /* partial completion */
1519                                 break;
1520                         rx_event = true;
1521                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1522                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1523                                     (TX_CMP_TYPE(txcmp) ==
1524                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1525                                     (TX_CMP_TYPE(txcmp) ==
1526                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1527                         bnxt_hwrm_handler(bp, txcmp);
1528                 }
1529                 raw_cons = NEXT_RAW_CMP(raw_cons);
1530
1531                 if (rx_pkts == budget)
1532                         break;
1533         }
1534
1535         cpr->cp_raw_cons = raw_cons;
1536         /* ACK completion ring before freeing tx ring and producing new
1537          * buffers in rx/agg rings to prevent overflowing the completion
1538          * ring.
1539          */
1540         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1541
1542         if (tx_pkts)
1543                 bnxt_tx_int(bp, bnapi, tx_pkts);
1544
1545         if (rx_event) {
1546                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1547
1548                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1549                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1550                 if (agg_event) {
1551                         writel(DB_KEY_RX | rxr->rx_agg_prod,
1552                                rxr->rx_agg_doorbell);
1553                         writel(DB_KEY_RX | rxr->rx_agg_prod,
1554                                rxr->rx_agg_doorbell);
1555                 }
1556         }
1557         return rx_pkts;
1558 }
1559
1560 static int bnxt_poll(struct napi_struct *napi, int budget)
1561 {
1562         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1563         struct bnxt *bp = bnapi->bp;
1564         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1565         int work_done = 0;
1566
1567         if (!bnxt_lock_napi(bnapi))
1568                 return budget;
1569
1570         while (1) {
1571                 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
1572
1573                 if (work_done >= budget)
1574                         break;
1575
1576                 if (!bnxt_has_work(bp, cpr)) {
1577                         napi_complete(napi);
1578                         BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1579                         break;
1580                 }
1581         }
1582         mmiowb();
1583         bnxt_unlock_napi(bnapi);
1584         return work_done;
1585 }
1586
1587 #ifdef CONFIG_NET_RX_BUSY_POLL
1588 static int bnxt_busy_poll(struct napi_struct *napi)
1589 {
1590         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1591         struct bnxt *bp = bnapi->bp;
1592         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1593         int rx_work, budget = 4;
1594
1595         if (atomic_read(&bp->intr_sem) != 0)
1596                 return LL_FLUSH_FAILED;
1597
1598         if (!bnxt_lock_poll(bnapi))
1599                 return LL_FLUSH_BUSY;
1600
1601         rx_work = bnxt_poll_work(bp, bnapi, budget);
1602
1603         BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1604
1605         bnxt_unlock_poll(bnapi);
1606         return rx_work;
1607 }
1608 #endif
1609
1610 static void bnxt_free_tx_skbs(struct bnxt *bp)
1611 {
1612         int i, max_idx;
1613         struct pci_dev *pdev = bp->pdev;
1614
1615         if (!bp->tx_ring)
1616                 return;
1617
1618         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
1619         for (i = 0; i < bp->tx_nr_rings; i++) {
1620                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1621                 int j;
1622
1623                 for (j = 0; j < max_idx;) {
1624                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
1625                         struct sk_buff *skb = tx_buf->skb;
1626                         int k, last;
1627
1628                         if (!skb) {
1629                                 j++;
1630                                 continue;
1631                         }
1632
1633                         tx_buf->skb = NULL;
1634
1635                         if (tx_buf->is_push) {
1636                                 dev_kfree_skb(skb);
1637                                 j += 2;
1638                                 continue;
1639                         }
1640
1641                         dma_unmap_single(&pdev->dev,
1642                                          dma_unmap_addr(tx_buf, mapping),
1643                                          skb_headlen(skb),
1644                                          PCI_DMA_TODEVICE);
1645
1646                         last = tx_buf->nr_frags;
1647                         j += 2;
1648                         for (k = 0; k < last; k++, j++) {
1649                                 int ring_idx = j & bp->tx_ring_mask;
1650                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
1651
1652                                 tx_buf = &txr->tx_buf_ring[ring_idx];
1653                                 dma_unmap_page(
1654                                         &pdev->dev,
1655                                         dma_unmap_addr(tx_buf, mapping),
1656                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
1657                         }
1658                         dev_kfree_skb(skb);
1659                 }
1660                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
1661         }
1662 }
1663
1664 static void bnxt_free_rx_skbs(struct bnxt *bp)
1665 {
1666         int i, max_idx, max_agg_idx;
1667         struct pci_dev *pdev = bp->pdev;
1668
1669         if (!bp->rx_ring)
1670                 return;
1671
1672         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
1673         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
1674         for (i = 0; i < bp->rx_nr_rings; i++) {
1675                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1676                 int j;
1677
1678                 if (rxr->rx_tpa) {
1679                         for (j = 0; j < MAX_TPA; j++) {
1680                                 struct bnxt_tpa_info *tpa_info =
1681                                                         &rxr->rx_tpa[j];
1682                                 u8 *data = tpa_info->data;
1683
1684                                 if (!data)
1685                                         continue;
1686
1687                                 dma_unmap_single(
1688                                         &pdev->dev,
1689                                         dma_unmap_addr(tpa_info, mapping),
1690                                         bp->rx_buf_use_size,
1691                                         PCI_DMA_FROMDEVICE);
1692
1693                                 tpa_info->data = NULL;
1694
1695                                 kfree(data);
1696                         }
1697                 }
1698
1699                 for (j = 0; j < max_idx; j++) {
1700                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
1701                         u8 *data = rx_buf->data;
1702
1703                         if (!data)
1704                                 continue;
1705
1706                         dma_unmap_single(&pdev->dev,
1707                                          dma_unmap_addr(rx_buf, mapping),
1708                                          bp->rx_buf_use_size,
1709                                          PCI_DMA_FROMDEVICE);
1710
1711                         rx_buf->data = NULL;
1712
1713                         kfree(data);
1714                 }
1715
1716                 for (j = 0; j < max_agg_idx; j++) {
1717                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
1718                                 &rxr->rx_agg_ring[j];
1719                         struct page *page = rx_agg_buf->page;
1720
1721                         if (!page)
1722                                 continue;
1723
1724                         dma_unmap_page(&pdev->dev,
1725                                        dma_unmap_addr(rx_agg_buf, mapping),
1726                                        BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE);
1727
1728                         rx_agg_buf->page = NULL;
1729                         __clear_bit(j, rxr->rx_agg_bmap);
1730
1731                         __free_page(page);
1732                 }
1733                 if (rxr->rx_page) {
1734                         __free_page(rxr->rx_page);
1735                         rxr->rx_page = NULL;
1736                 }
1737         }
1738 }
1739
1740 static void bnxt_free_skbs(struct bnxt *bp)
1741 {
1742         bnxt_free_tx_skbs(bp);
1743         bnxt_free_rx_skbs(bp);
1744 }
1745
1746 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1747 {
1748         struct pci_dev *pdev = bp->pdev;
1749         int i;
1750
1751         for (i = 0; i < ring->nr_pages; i++) {
1752                 if (!ring->pg_arr[i])
1753                         continue;
1754
1755                 dma_free_coherent(&pdev->dev, ring->page_size,
1756                                   ring->pg_arr[i], ring->dma_arr[i]);
1757
1758                 ring->pg_arr[i] = NULL;
1759         }
1760         if (ring->pg_tbl) {
1761                 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
1762                                   ring->pg_tbl, ring->pg_tbl_map);
1763                 ring->pg_tbl = NULL;
1764         }
1765         if (ring->vmem_size && *ring->vmem) {
1766                 vfree(*ring->vmem);
1767                 *ring->vmem = NULL;
1768         }
1769 }
1770
1771 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1772 {
1773         int i;
1774         struct pci_dev *pdev = bp->pdev;
1775
1776         if (ring->nr_pages > 1) {
1777                 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
1778                                                   ring->nr_pages * 8,
1779                                                   &ring->pg_tbl_map,
1780                                                   GFP_KERNEL);
1781                 if (!ring->pg_tbl)
1782                         return -ENOMEM;
1783         }
1784
1785         for (i = 0; i < ring->nr_pages; i++) {
1786                 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
1787                                                      ring->page_size,
1788                                                      &ring->dma_arr[i],
1789                                                      GFP_KERNEL);
1790                 if (!ring->pg_arr[i])
1791                         return -ENOMEM;
1792
1793                 if (ring->nr_pages > 1)
1794                         ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
1795         }
1796
1797         if (ring->vmem_size) {
1798                 *ring->vmem = vzalloc(ring->vmem_size);
1799                 if (!(*ring->vmem))
1800                         return -ENOMEM;
1801         }
1802         return 0;
1803 }
1804
1805 static void bnxt_free_rx_rings(struct bnxt *bp)
1806 {
1807         int i;
1808
1809         if (!bp->rx_ring)
1810                 return;
1811
1812         for (i = 0; i < bp->rx_nr_rings; i++) {
1813                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1814                 struct bnxt_ring_struct *ring;
1815
1816                 kfree(rxr->rx_tpa);
1817                 rxr->rx_tpa = NULL;
1818
1819                 kfree(rxr->rx_agg_bmap);
1820                 rxr->rx_agg_bmap = NULL;
1821
1822                 ring = &rxr->rx_ring_struct;
1823                 bnxt_free_ring(bp, ring);
1824
1825                 ring = &rxr->rx_agg_ring_struct;
1826                 bnxt_free_ring(bp, ring);
1827         }
1828 }
1829
1830 static int bnxt_alloc_rx_rings(struct bnxt *bp)
1831 {
1832         int i, rc, agg_rings = 0, tpa_rings = 0;
1833
1834         if (!bp->rx_ring)
1835                 return -ENOMEM;
1836
1837         if (bp->flags & BNXT_FLAG_AGG_RINGS)
1838                 agg_rings = 1;
1839
1840         if (bp->flags & BNXT_FLAG_TPA)
1841                 tpa_rings = 1;
1842
1843         for (i = 0; i < bp->rx_nr_rings; i++) {
1844                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1845                 struct bnxt_ring_struct *ring;
1846
1847                 ring = &rxr->rx_ring_struct;
1848
1849                 rc = bnxt_alloc_ring(bp, ring);
1850                 if (rc)
1851                         return rc;
1852
1853                 if (agg_rings) {
1854                         u16 mem_size;
1855
1856                         ring = &rxr->rx_agg_ring_struct;
1857                         rc = bnxt_alloc_ring(bp, ring);
1858                         if (rc)
1859                                 return rc;
1860
1861                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
1862                         mem_size = rxr->rx_agg_bmap_size / 8;
1863                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
1864                         if (!rxr->rx_agg_bmap)
1865                                 return -ENOMEM;
1866
1867                         if (tpa_rings) {
1868                                 rxr->rx_tpa = kcalloc(MAX_TPA,
1869                                                 sizeof(struct bnxt_tpa_info),
1870                                                 GFP_KERNEL);
1871                                 if (!rxr->rx_tpa)
1872                                         return -ENOMEM;
1873                         }
1874                 }
1875         }
1876         return 0;
1877 }
1878
1879 static void bnxt_free_tx_rings(struct bnxt *bp)
1880 {
1881         int i;
1882         struct pci_dev *pdev = bp->pdev;
1883
1884         if (!bp->tx_ring)
1885                 return;
1886
1887         for (i = 0; i < bp->tx_nr_rings; i++) {
1888                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1889                 struct bnxt_ring_struct *ring;
1890
1891                 if (txr->tx_push) {
1892                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
1893                                           txr->tx_push, txr->tx_push_mapping);
1894                         txr->tx_push = NULL;
1895                 }
1896
1897                 ring = &txr->tx_ring_struct;
1898
1899                 bnxt_free_ring(bp, ring);
1900         }
1901 }
1902
1903 static int bnxt_alloc_tx_rings(struct bnxt *bp)
1904 {
1905         int i, j, rc;
1906         struct pci_dev *pdev = bp->pdev;
1907
1908         bp->tx_push_size = 0;
1909         if (bp->tx_push_thresh) {
1910                 int push_size;
1911
1912                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
1913                                         bp->tx_push_thresh);
1914
1915                 if (push_size > 256) {
1916                         push_size = 0;
1917                         bp->tx_push_thresh = 0;
1918                 }
1919
1920                 bp->tx_push_size = push_size;
1921         }
1922
1923         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
1924                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1925                 struct bnxt_ring_struct *ring;
1926
1927                 ring = &txr->tx_ring_struct;
1928
1929                 rc = bnxt_alloc_ring(bp, ring);
1930                 if (rc)
1931                         return rc;
1932
1933                 if (bp->tx_push_size) {
1934                         dma_addr_t mapping;
1935
1936                         /* One pre-allocated DMA buffer to backup
1937                          * TX push operation
1938                          */
1939                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
1940                                                 bp->tx_push_size,
1941                                                 &txr->tx_push_mapping,
1942                                                 GFP_KERNEL);
1943
1944                         if (!txr->tx_push)
1945                                 return -ENOMEM;
1946
1947                         mapping = txr->tx_push_mapping +
1948                                 sizeof(struct tx_push_bd);
1949                         txr->data_mapping = cpu_to_le64(mapping);
1950
1951                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
1952                 }
1953                 ring->queue_id = bp->q_info[j].queue_id;
1954                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
1955                         j++;
1956         }
1957         return 0;
1958 }
1959
1960 static void bnxt_free_cp_rings(struct bnxt *bp)
1961 {
1962         int i;
1963
1964         if (!bp->bnapi)
1965                 return;
1966
1967         for (i = 0; i < bp->cp_nr_rings; i++) {
1968                 struct bnxt_napi *bnapi = bp->bnapi[i];
1969                 struct bnxt_cp_ring_info *cpr;
1970                 struct bnxt_ring_struct *ring;
1971
1972                 if (!bnapi)
1973                         continue;
1974
1975                 cpr = &bnapi->cp_ring;
1976                 ring = &cpr->cp_ring_struct;
1977
1978                 bnxt_free_ring(bp, ring);
1979         }
1980 }
1981
1982 static int bnxt_alloc_cp_rings(struct bnxt *bp)
1983 {
1984         int i, rc;
1985
1986         for (i = 0; i < bp->cp_nr_rings; i++) {
1987                 struct bnxt_napi *bnapi = bp->bnapi[i];
1988                 struct bnxt_cp_ring_info *cpr;
1989                 struct bnxt_ring_struct *ring;
1990
1991                 if (!bnapi)
1992                         continue;
1993
1994                 cpr = &bnapi->cp_ring;
1995                 ring = &cpr->cp_ring_struct;
1996
1997                 rc = bnxt_alloc_ring(bp, ring);
1998                 if (rc)
1999                         return rc;
2000         }
2001         return 0;
2002 }
2003
2004 static void bnxt_init_ring_struct(struct bnxt *bp)
2005 {
2006         int i;
2007
2008         for (i = 0; i < bp->cp_nr_rings; i++) {
2009                 struct bnxt_napi *bnapi = bp->bnapi[i];
2010                 struct bnxt_cp_ring_info *cpr;
2011                 struct bnxt_rx_ring_info *rxr;
2012                 struct bnxt_tx_ring_info *txr;
2013                 struct bnxt_ring_struct *ring;
2014
2015                 if (!bnapi)
2016                         continue;
2017
2018                 cpr = &bnapi->cp_ring;
2019                 ring = &cpr->cp_ring_struct;
2020                 ring->nr_pages = bp->cp_nr_pages;
2021                 ring->page_size = HW_CMPD_RING_SIZE;
2022                 ring->pg_arr = (void **)cpr->cp_desc_ring;
2023                 ring->dma_arr = cpr->cp_desc_mapping;
2024                 ring->vmem_size = 0;
2025
2026                 rxr = bnapi->rx_ring;
2027                 if (!rxr)
2028                         goto skip_rx;
2029
2030                 ring = &rxr->rx_ring_struct;
2031                 ring->nr_pages = bp->rx_nr_pages;
2032                 ring->page_size = HW_RXBD_RING_SIZE;
2033                 ring->pg_arr = (void **)rxr->rx_desc_ring;
2034                 ring->dma_arr = rxr->rx_desc_mapping;
2035                 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2036                 ring->vmem = (void **)&rxr->rx_buf_ring;
2037
2038                 ring = &rxr->rx_agg_ring_struct;
2039                 ring->nr_pages = bp->rx_agg_nr_pages;
2040                 ring->page_size = HW_RXBD_RING_SIZE;
2041                 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2042                 ring->dma_arr = rxr->rx_agg_desc_mapping;
2043                 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2044                 ring->vmem = (void **)&rxr->rx_agg_ring;
2045
2046 skip_rx:
2047                 txr = bnapi->tx_ring;
2048                 if (!txr)
2049                         continue;
2050
2051                 ring = &txr->tx_ring_struct;
2052                 ring->nr_pages = bp->tx_nr_pages;
2053                 ring->page_size = HW_RXBD_RING_SIZE;
2054                 ring->pg_arr = (void **)txr->tx_desc_ring;
2055                 ring->dma_arr = txr->tx_desc_mapping;
2056                 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2057                 ring->vmem = (void **)&txr->tx_buf_ring;
2058         }
2059 }
2060
2061 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2062 {
2063         int i;
2064         u32 prod;
2065         struct rx_bd **rx_buf_ring;
2066
2067         rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2068         for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2069                 int j;
2070                 struct rx_bd *rxbd;
2071
2072                 rxbd = rx_buf_ring[i];
2073                 if (!rxbd)
2074                         continue;
2075
2076                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2077                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2078                         rxbd->rx_bd_opaque = prod;
2079                 }
2080         }
2081 }
2082
2083 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2084 {
2085         struct net_device *dev = bp->dev;
2086         struct bnxt_rx_ring_info *rxr;
2087         struct bnxt_ring_struct *ring;
2088         u32 prod, type;
2089         int i;
2090
2091         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2092                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2093
2094         if (NET_IP_ALIGN == 2)
2095                 type |= RX_BD_FLAGS_SOP;
2096
2097         rxr = &bp->rx_ring[ring_nr];
2098         ring = &rxr->rx_ring_struct;
2099         bnxt_init_rxbd_pages(ring, type);
2100
2101         prod = rxr->rx_prod;
2102         for (i = 0; i < bp->rx_ring_size; i++) {
2103                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2104                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2105                                     ring_nr, i, bp->rx_ring_size);
2106                         break;
2107                 }
2108                 prod = NEXT_RX(prod);
2109         }
2110         rxr->rx_prod = prod;
2111         ring->fw_ring_id = INVALID_HW_RING_ID;
2112
2113         ring = &rxr->rx_agg_ring_struct;
2114         ring->fw_ring_id = INVALID_HW_RING_ID;
2115
2116         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2117                 return 0;
2118
2119         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2120                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2121
2122         bnxt_init_rxbd_pages(ring, type);
2123
2124         prod = rxr->rx_agg_prod;
2125         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2126                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2127                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2128                                     ring_nr, i, bp->rx_ring_size);
2129                         break;
2130                 }
2131                 prod = NEXT_RX_AGG(prod);
2132         }
2133         rxr->rx_agg_prod = prod;
2134
2135         if (bp->flags & BNXT_FLAG_TPA) {
2136                 if (rxr->rx_tpa) {
2137                         u8 *data;
2138                         dma_addr_t mapping;
2139
2140                         for (i = 0; i < MAX_TPA; i++) {
2141                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2142                                                             GFP_KERNEL);
2143                                 if (!data)
2144                                         return -ENOMEM;
2145
2146                                 rxr->rx_tpa[i].data = data;
2147                                 rxr->rx_tpa[i].mapping = mapping;
2148                         }
2149                 } else {
2150                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2151                         return -ENOMEM;
2152                 }
2153         }
2154
2155         return 0;
2156 }
2157
2158 static int bnxt_init_rx_rings(struct bnxt *bp)
2159 {
2160         int i, rc = 0;
2161
2162         for (i = 0; i < bp->rx_nr_rings; i++) {
2163                 rc = bnxt_init_one_rx_ring(bp, i);
2164                 if (rc)
2165                         break;
2166         }
2167
2168         return rc;
2169 }
2170
2171 static int bnxt_init_tx_rings(struct bnxt *bp)
2172 {
2173         u16 i;
2174
2175         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2176                                    MAX_SKB_FRAGS + 1);
2177
2178         for (i = 0; i < bp->tx_nr_rings; i++) {
2179                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2180                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2181
2182                 ring->fw_ring_id = INVALID_HW_RING_ID;
2183         }
2184
2185         return 0;
2186 }
2187
2188 static void bnxt_free_ring_grps(struct bnxt *bp)
2189 {
2190         kfree(bp->grp_info);
2191         bp->grp_info = NULL;
2192 }
2193
2194 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2195 {
2196         int i;
2197
2198         if (irq_re_init) {
2199                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2200                                        sizeof(struct bnxt_ring_grp_info),
2201                                        GFP_KERNEL);
2202                 if (!bp->grp_info)
2203                         return -ENOMEM;
2204         }
2205         for (i = 0; i < bp->cp_nr_rings; i++) {
2206                 if (irq_re_init)
2207                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2208                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2209                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2210                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2211                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2212         }
2213         return 0;
2214 }
2215
2216 static void bnxt_free_vnics(struct bnxt *bp)
2217 {
2218         kfree(bp->vnic_info);
2219         bp->vnic_info = NULL;
2220         bp->nr_vnics = 0;
2221 }
2222
2223 static int bnxt_alloc_vnics(struct bnxt *bp)
2224 {
2225         int num_vnics = 1;
2226
2227 #ifdef CONFIG_RFS_ACCEL
2228         if (bp->flags & BNXT_FLAG_RFS)
2229                 num_vnics += bp->rx_nr_rings;
2230 #endif
2231
2232         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2233                                 GFP_KERNEL);
2234         if (!bp->vnic_info)
2235                 return -ENOMEM;
2236
2237         bp->nr_vnics = num_vnics;
2238         return 0;
2239 }
2240
2241 static void bnxt_init_vnics(struct bnxt *bp)
2242 {
2243         int i;
2244
2245         for (i = 0; i < bp->nr_vnics; i++) {
2246                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2247
2248                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2249                 vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
2250                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2251
2252                 if (bp->vnic_info[i].rss_hash_key) {
2253                         if (i == 0)
2254                                 prandom_bytes(vnic->rss_hash_key,
2255                                               HW_HASH_KEY_SIZE);
2256                         else
2257                                 memcpy(vnic->rss_hash_key,
2258                                        bp->vnic_info[0].rss_hash_key,
2259                                        HW_HASH_KEY_SIZE);
2260                 }
2261         }
2262 }
2263
2264 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2265 {
2266         int pages;
2267
2268         pages = ring_size / desc_per_pg;
2269
2270         if (!pages)
2271                 return 1;
2272
2273         pages++;
2274
2275         while (pages & (pages - 1))
2276                 pages++;
2277
2278         return pages;
2279 }
2280
2281 static void bnxt_set_tpa_flags(struct bnxt *bp)
2282 {
2283         bp->flags &= ~BNXT_FLAG_TPA;
2284         if (bp->dev->features & NETIF_F_LRO)
2285                 bp->flags |= BNXT_FLAG_LRO;
2286         if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
2287                 bp->flags |= BNXT_FLAG_GRO;
2288 }
2289
2290 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2291  * be set on entry.
2292  */
2293 void bnxt_set_ring_params(struct bnxt *bp)
2294 {
2295         u32 ring_size, rx_size, rx_space;
2296         u32 agg_factor = 0, agg_ring_size = 0;
2297
2298         /* 8 for CRC and VLAN */
2299         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2300
2301         rx_space = rx_size + NET_SKB_PAD +
2302                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2303
2304         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2305         ring_size = bp->rx_ring_size;
2306         bp->rx_agg_ring_size = 0;
2307         bp->rx_agg_nr_pages = 0;
2308
2309         if (bp->flags & BNXT_FLAG_TPA)
2310                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
2311
2312         bp->flags &= ~BNXT_FLAG_JUMBO;
2313         if (rx_space > PAGE_SIZE) {
2314                 u32 jumbo_factor;
2315
2316                 bp->flags |= BNXT_FLAG_JUMBO;
2317                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2318                 if (jumbo_factor > agg_factor)
2319                         agg_factor = jumbo_factor;
2320         }
2321         agg_ring_size = ring_size * agg_factor;
2322
2323         if (agg_ring_size) {
2324                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2325                                                         RX_DESC_CNT);
2326                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2327                         u32 tmp = agg_ring_size;
2328
2329                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2330                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2331                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2332                                     tmp, agg_ring_size);
2333                 }
2334                 bp->rx_agg_ring_size = agg_ring_size;
2335                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2336                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2337                 rx_space = rx_size + NET_SKB_PAD +
2338                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2339         }
2340
2341         bp->rx_buf_use_size = rx_size;
2342         bp->rx_buf_size = rx_space;
2343
2344         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2345         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2346
2347         ring_size = bp->tx_ring_size;
2348         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2349         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2350
2351         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2352         bp->cp_ring_size = ring_size;
2353
2354         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2355         if (bp->cp_nr_pages > MAX_CP_PAGES) {
2356                 bp->cp_nr_pages = MAX_CP_PAGES;
2357                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2358                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2359                             ring_size, bp->cp_ring_size);
2360         }
2361         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2362         bp->cp_ring_mask = bp->cp_bit - 1;
2363 }
2364
2365 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2366 {
2367         int i;
2368         struct bnxt_vnic_info *vnic;
2369         struct pci_dev *pdev = bp->pdev;
2370
2371         if (!bp->vnic_info)
2372                 return;
2373
2374         for (i = 0; i < bp->nr_vnics; i++) {
2375                 vnic = &bp->vnic_info[i];
2376
2377                 kfree(vnic->fw_grp_ids);
2378                 vnic->fw_grp_ids = NULL;
2379
2380                 kfree(vnic->uc_list);
2381                 vnic->uc_list = NULL;
2382
2383                 if (vnic->mc_list) {
2384                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2385                                           vnic->mc_list, vnic->mc_list_mapping);
2386                         vnic->mc_list = NULL;
2387                 }
2388
2389                 if (vnic->rss_table) {
2390                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
2391                                           vnic->rss_table,
2392                                           vnic->rss_table_dma_addr);
2393                         vnic->rss_table = NULL;
2394                 }
2395
2396                 vnic->rss_hash_key = NULL;
2397                 vnic->flags = 0;
2398         }
2399 }
2400
2401 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2402 {
2403         int i, rc = 0, size;
2404         struct bnxt_vnic_info *vnic;
2405         struct pci_dev *pdev = bp->pdev;
2406         int max_rings;
2407
2408         for (i = 0; i < bp->nr_vnics; i++) {
2409                 vnic = &bp->vnic_info[i];
2410
2411                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2412                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2413
2414                         if (mem_size > 0) {
2415                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2416                                 if (!vnic->uc_list) {
2417                                         rc = -ENOMEM;
2418                                         goto out;
2419                                 }
2420                         }
2421                 }
2422
2423                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2424                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2425                         vnic->mc_list =
2426                                 dma_alloc_coherent(&pdev->dev,
2427                                                    vnic->mc_list_size,
2428                                                    &vnic->mc_list_mapping,
2429                                                    GFP_KERNEL);
2430                         if (!vnic->mc_list) {
2431                                 rc = -ENOMEM;
2432                                 goto out;
2433                         }
2434                 }
2435
2436                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2437                         max_rings = bp->rx_nr_rings;
2438                 else
2439                         max_rings = 1;
2440
2441                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2442                 if (!vnic->fw_grp_ids) {
2443                         rc = -ENOMEM;
2444                         goto out;
2445                 }
2446
2447                 /* Allocate rss table and hash key */
2448                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2449                                                      &vnic->rss_table_dma_addr,
2450                                                      GFP_KERNEL);
2451                 if (!vnic->rss_table) {
2452                         rc = -ENOMEM;
2453                         goto out;
2454                 }
2455
2456                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2457
2458                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2459                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2460         }
2461         return 0;
2462
2463 out:
2464         return rc;
2465 }
2466
2467 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2468 {
2469         struct pci_dev *pdev = bp->pdev;
2470
2471         dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2472                           bp->hwrm_cmd_resp_dma_addr);
2473
2474         bp->hwrm_cmd_resp_addr = NULL;
2475         if (bp->hwrm_dbg_resp_addr) {
2476                 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2477                                   bp->hwrm_dbg_resp_addr,
2478                                   bp->hwrm_dbg_resp_dma_addr);
2479
2480                 bp->hwrm_dbg_resp_addr = NULL;
2481         }
2482 }
2483
2484 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2485 {
2486         struct pci_dev *pdev = bp->pdev;
2487
2488         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2489                                                    &bp->hwrm_cmd_resp_dma_addr,
2490                                                    GFP_KERNEL);
2491         if (!bp->hwrm_cmd_resp_addr)
2492                 return -ENOMEM;
2493         bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2494                                                     HWRM_DBG_REG_BUF_SIZE,
2495                                                     &bp->hwrm_dbg_resp_dma_addr,
2496                                                     GFP_KERNEL);
2497         if (!bp->hwrm_dbg_resp_addr)
2498                 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2499
2500         return 0;
2501 }
2502
2503 static void bnxt_free_stats(struct bnxt *bp)
2504 {
2505         u32 size, i;
2506         struct pci_dev *pdev = bp->pdev;
2507
2508         if (bp->hw_rx_port_stats) {
2509                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
2510                                   bp->hw_rx_port_stats,
2511                                   bp->hw_rx_port_stats_map);
2512                 bp->hw_rx_port_stats = NULL;
2513                 bp->flags &= ~BNXT_FLAG_PORT_STATS;
2514         }
2515
2516         if (!bp->bnapi)
2517                 return;
2518
2519         size = sizeof(struct ctx_hw_stats);
2520
2521         for (i = 0; i < bp->cp_nr_rings; i++) {
2522                 struct bnxt_napi *bnapi = bp->bnapi[i];
2523                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2524
2525                 if (cpr->hw_stats) {
2526                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
2527                                           cpr->hw_stats_map);
2528                         cpr->hw_stats = NULL;
2529                 }
2530         }
2531 }
2532
2533 static int bnxt_alloc_stats(struct bnxt *bp)
2534 {
2535         u32 size, i;
2536         struct pci_dev *pdev = bp->pdev;
2537
2538         size = sizeof(struct ctx_hw_stats);
2539
2540         for (i = 0; i < bp->cp_nr_rings; i++) {
2541                 struct bnxt_napi *bnapi = bp->bnapi[i];
2542                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2543
2544                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
2545                                                    &cpr->hw_stats_map,
2546                                                    GFP_KERNEL);
2547                 if (!cpr->hw_stats)
2548                         return -ENOMEM;
2549
2550                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
2551         }
2552
2553         if (BNXT_PF(bp)) {
2554                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
2555                                          sizeof(struct tx_port_stats) + 1024;
2556
2557                 bp->hw_rx_port_stats =
2558                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
2559                                            &bp->hw_rx_port_stats_map,
2560                                            GFP_KERNEL);
2561                 if (!bp->hw_rx_port_stats)
2562                         return -ENOMEM;
2563
2564                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
2565                                        512;
2566                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
2567                                            sizeof(struct rx_port_stats) + 512;
2568                 bp->flags |= BNXT_FLAG_PORT_STATS;
2569         }
2570         return 0;
2571 }
2572
2573 static void bnxt_clear_ring_indices(struct bnxt *bp)
2574 {
2575         int i;
2576
2577         if (!bp->bnapi)
2578                 return;
2579
2580         for (i = 0; i < bp->cp_nr_rings; i++) {
2581                 struct bnxt_napi *bnapi = bp->bnapi[i];
2582                 struct bnxt_cp_ring_info *cpr;
2583                 struct bnxt_rx_ring_info *rxr;
2584                 struct bnxt_tx_ring_info *txr;
2585
2586                 if (!bnapi)
2587                         continue;
2588
2589                 cpr = &bnapi->cp_ring;
2590                 cpr->cp_raw_cons = 0;
2591
2592                 txr = bnapi->tx_ring;
2593                 if (txr) {
2594                         txr->tx_prod = 0;
2595                         txr->tx_cons = 0;
2596                 }
2597
2598                 rxr = bnapi->rx_ring;
2599                 if (rxr) {
2600                         rxr->rx_prod = 0;
2601                         rxr->rx_agg_prod = 0;
2602                         rxr->rx_sw_agg_prod = 0;
2603                         rxr->rx_next_cons = 0;
2604                 }
2605         }
2606 }
2607
2608 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
2609 {
2610 #ifdef CONFIG_RFS_ACCEL
2611         int i;
2612
2613         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
2614          * safe to delete the hash table.
2615          */
2616         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
2617                 struct hlist_head *head;
2618                 struct hlist_node *tmp;
2619                 struct bnxt_ntuple_filter *fltr;
2620
2621                 head = &bp->ntp_fltr_hash_tbl[i];
2622                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
2623                         hlist_del(&fltr->hash);
2624                         kfree(fltr);
2625                 }
2626         }
2627         if (irq_reinit) {
2628                 kfree(bp->ntp_fltr_bmap);
2629                 bp->ntp_fltr_bmap = NULL;
2630         }
2631         bp->ntp_fltr_count = 0;
2632 #endif
2633 }
2634
2635 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
2636 {
2637 #ifdef CONFIG_RFS_ACCEL
2638         int i, rc = 0;
2639
2640         if (!(bp->flags & BNXT_FLAG_RFS))
2641                 return 0;
2642
2643         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
2644                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
2645
2646         bp->ntp_fltr_count = 0;
2647         bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
2648                                     GFP_KERNEL);
2649
2650         if (!bp->ntp_fltr_bmap)
2651                 rc = -ENOMEM;
2652
2653         return rc;
2654 #else
2655         return 0;
2656 #endif
2657 }
2658
2659 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
2660 {
2661         bnxt_free_vnic_attributes(bp);
2662         bnxt_free_tx_rings(bp);
2663         bnxt_free_rx_rings(bp);
2664         bnxt_free_cp_rings(bp);
2665         bnxt_free_ntp_fltrs(bp, irq_re_init);
2666         if (irq_re_init) {
2667                 bnxt_free_stats(bp);
2668                 bnxt_free_ring_grps(bp);
2669                 bnxt_free_vnics(bp);
2670                 kfree(bp->tx_ring);
2671                 bp->tx_ring = NULL;
2672                 kfree(bp->rx_ring);
2673                 bp->rx_ring = NULL;
2674                 kfree(bp->bnapi);
2675                 bp->bnapi = NULL;
2676         } else {
2677                 bnxt_clear_ring_indices(bp);
2678         }
2679 }
2680
2681 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
2682 {
2683         int i, j, rc, size, arr_size;
2684         void *bnapi;
2685
2686         if (irq_re_init) {
2687                 /* Allocate bnapi mem pointer array and mem block for
2688                  * all queues
2689                  */
2690                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
2691                                 bp->cp_nr_rings);
2692                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
2693                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
2694                 if (!bnapi)
2695                         return -ENOMEM;
2696
2697                 bp->bnapi = bnapi;
2698                 bnapi += arr_size;
2699                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
2700                         bp->bnapi[i] = bnapi;
2701                         bp->bnapi[i]->index = i;
2702                         bp->bnapi[i]->bp = bp;
2703                 }
2704
2705                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
2706                                       sizeof(struct bnxt_rx_ring_info),
2707                                       GFP_KERNEL);
2708                 if (!bp->rx_ring)
2709                         return -ENOMEM;
2710
2711                 for (i = 0; i < bp->rx_nr_rings; i++) {
2712                         bp->rx_ring[i].bnapi = bp->bnapi[i];
2713                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
2714                 }
2715
2716                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
2717                                       sizeof(struct bnxt_tx_ring_info),
2718                                       GFP_KERNEL);
2719                 if (!bp->tx_ring)
2720                         return -ENOMEM;
2721
2722                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
2723                         j = 0;
2724                 else
2725                         j = bp->rx_nr_rings;
2726
2727                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
2728                         bp->tx_ring[i].bnapi = bp->bnapi[j];
2729                         bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
2730                 }
2731
2732                 rc = bnxt_alloc_stats(bp);
2733                 if (rc)
2734                         goto alloc_mem_err;
2735
2736                 rc = bnxt_alloc_ntp_fltrs(bp);
2737                 if (rc)
2738                         goto alloc_mem_err;
2739
2740                 rc = bnxt_alloc_vnics(bp);
2741                 if (rc)
2742                         goto alloc_mem_err;
2743         }
2744
2745         bnxt_init_ring_struct(bp);
2746
2747         rc = bnxt_alloc_rx_rings(bp);
2748         if (rc)
2749                 goto alloc_mem_err;
2750
2751         rc = bnxt_alloc_tx_rings(bp);
2752         if (rc)
2753                 goto alloc_mem_err;
2754
2755         rc = bnxt_alloc_cp_rings(bp);
2756         if (rc)
2757                 goto alloc_mem_err;
2758
2759         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
2760                                   BNXT_VNIC_UCAST_FLAG;
2761         rc = bnxt_alloc_vnic_attributes(bp);
2762         if (rc)
2763                 goto alloc_mem_err;
2764         return 0;
2765
2766 alloc_mem_err:
2767         bnxt_free_mem(bp, true);
2768         return rc;
2769 }
2770
2771 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
2772                             u16 cmpl_ring, u16 target_id)
2773 {
2774         struct input *req = request;
2775
2776         req->req_type = cpu_to_le16(req_type);
2777         req->cmpl_ring = cpu_to_le16(cmpl_ring);
2778         req->target_id = cpu_to_le16(target_id);
2779         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
2780 }
2781
2782 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
2783                                  int timeout, bool silent)
2784 {
2785         int i, intr_process, rc, tmo_count;
2786         struct input *req = msg;
2787         u32 *data = msg;
2788         __le32 *resp_len, *valid;
2789         u16 cp_ring_id, len = 0;
2790         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
2791
2792         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
2793         memset(resp, 0, PAGE_SIZE);
2794         cp_ring_id = le16_to_cpu(req->cmpl_ring);
2795         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
2796
2797         /* Write request msg to hwrm channel */
2798         __iowrite32_copy(bp->bar0, data, msg_len / 4);
2799
2800         for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4)
2801                 writel(0, bp->bar0 + i);
2802
2803         /* currently supports only one outstanding message */
2804         if (intr_process)
2805                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
2806
2807         /* Ring channel doorbell */
2808         writel(1, bp->bar0 + 0x100);
2809
2810         if (!timeout)
2811                 timeout = DFLT_HWRM_CMD_TIMEOUT;
2812
2813         i = 0;
2814         tmo_count = timeout * 40;
2815         if (intr_process) {
2816                 /* Wait until hwrm response cmpl interrupt is processed */
2817                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
2818                        i++ < tmo_count) {
2819                         usleep_range(25, 40);
2820                 }
2821
2822                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
2823                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
2824                                    le16_to_cpu(req->req_type));
2825                         return -1;
2826                 }
2827         } else {
2828                 /* Check if response len is updated */
2829                 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
2830                 for (i = 0; i < tmo_count; i++) {
2831                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
2832                               HWRM_RESP_LEN_SFT;
2833                         if (len)
2834                                 break;
2835                         usleep_range(25, 40);
2836                 }
2837
2838                 if (i >= tmo_count) {
2839                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
2840                                    timeout, le16_to_cpu(req->req_type),
2841                                    le16_to_cpu(req->seq_id), len);
2842                         return -1;
2843                 }
2844
2845                 /* Last word of resp contains valid bit */
2846                 valid = bp->hwrm_cmd_resp_addr + len - 4;
2847                 for (i = 0; i < 5; i++) {
2848                         if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
2849                                 break;
2850                         udelay(1);
2851                 }
2852
2853                 if (i >= 5) {
2854                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
2855                                    timeout, le16_to_cpu(req->req_type),
2856                                    le16_to_cpu(req->seq_id), len, *valid);
2857                         return -1;
2858                 }
2859         }
2860
2861         rc = le16_to_cpu(resp->error_code);
2862         if (rc && !silent)
2863                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
2864                            le16_to_cpu(resp->req_type),
2865                            le16_to_cpu(resp->seq_id), rc);
2866         return rc;
2867 }
2868
2869 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2870 {
2871         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
2872 }
2873
2874 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2875 {
2876         int rc;
2877
2878         mutex_lock(&bp->hwrm_cmd_lock);
2879         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
2880         mutex_unlock(&bp->hwrm_cmd_lock);
2881         return rc;
2882 }
2883
2884 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
2885                              int timeout)
2886 {
2887         int rc;
2888
2889         mutex_lock(&bp->hwrm_cmd_lock);
2890         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
2891         mutex_unlock(&bp->hwrm_cmd_lock);
2892         return rc;
2893 }
2894
2895 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
2896 {
2897         struct hwrm_func_drv_rgtr_input req = {0};
2898         int i;
2899         DECLARE_BITMAP(async_events_bmap, 256);
2900         u32 *events = (u32 *)async_events_bmap;
2901
2902         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
2903
2904         req.enables =
2905                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
2906                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
2907                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
2908
2909         memset(async_events_bmap, 0, sizeof(async_events_bmap));
2910         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
2911                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
2912
2913         for (i = 0; i < 8; i++)
2914                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
2915
2916         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
2917         req.ver_maj = DRV_VER_MAJ;
2918         req.ver_min = DRV_VER_MIN;
2919         req.ver_upd = DRV_VER_UPD;
2920
2921         if (BNXT_PF(bp)) {
2922                 DECLARE_BITMAP(vf_req_snif_bmap, 256);
2923                 u32 *data = (u32 *)vf_req_snif_bmap;
2924
2925                 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap));
2926                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++)
2927                         __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap);
2928
2929                 for (i = 0; i < 8; i++)
2930                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
2931
2932                 req.enables |=
2933                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
2934         }
2935
2936         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2937 }
2938
2939 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
2940 {
2941         struct hwrm_func_drv_unrgtr_input req = {0};
2942
2943         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
2944         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2945 }
2946
2947 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
2948 {
2949         u32 rc = 0;
2950         struct hwrm_tunnel_dst_port_free_input req = {0};
2951
2952         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
2953         req.tunnel_type = tunnel_type;
2954
2955         switch (tunnel_type) {
2956         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
2957                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
2958                 break;
2959         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
2960                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
2961                 break;
2962         default:
2963                 break;
2964         }
2965
2966         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2967         if (rc)
2968                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
2969                            rc);
2970         return rc;
2971 }
2972
2973 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
2974                                            u8 tunnel_type)
2975 {
2976         u32 rc = 0;
2977         struct hwrm_tunnel_dst_port_alloc_input req = {0};
2978         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2979
2980         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
2981
2982         req.tunnel_type = tunnel_type;
2983         req.tunnel_dst_port_val = port;
2984
2985         mutex_lock(&bp->hwrm_cmd_lock);
2986         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2987         if (rc) {
2988                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
2989                            rc);
2990                 goto err_out;
2991         }
2992
2993         if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN)
2994                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
2995
2996         else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE)
2997                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
2998 err_out:
2999         mutex_unlock(&bp->hwrm_cmd_lock);
3000         return rc;
3001 }
3002
3003 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3004 {
3005         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3006         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3007
3008         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
3009         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3010
3011         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3012         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3013         req.mask = cpu_to_le32(vnic->rx_mask);
3014         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3015 }
3016
3017 #ifdef CONFIG_RFS_ACCEL
3018 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3019                                             struct bnxt_ntuple_filter *fltr)
3020 {
3021         struct hwrm_cfa_ntuple_filter_free_input req = {0};
3022
3023         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3024         req.ntuple_filter_id = fltr->filter_id;
3025         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3026 }
3027
3028 #define BNXT_NTP_FLTR_FLAGS                                     \
3029         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
3030          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
3031          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
3032          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
3033          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
3034          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
3035          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
3036          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
3037          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
3038          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
3039          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
3040          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
3041          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
3042          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
3043
3044 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3045                                              struct bnxt_ntuple_filter *fltr)
3046 {
3047         int rc = 0;
3048         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3049         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3050                 bp->hwrm_cmd_resp_addr;
3051         struct flow_keys *keys = &fltr->fkeys;
3052         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3053
3054         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3055         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[0];
3056
3057         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3058
3059         req.ethertype = htons(ETH_P_IP);
3060         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
3061         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
3062         req.ip_protocol = keys->basic.ip_proto;
3063
3064         req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3065         req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3066         req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3067         req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3068
3069         req.src_port = keys->ports.src;
3070         req.src_port_mask = cpu_to_be16(0xffff);
3071         req.dst_port = keys->ports.dst;
3072         req.dst_port_mask = cpu_to_be16(0xffff);
3073
3074         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
3075         mutex_lock(&bp->hwrm_cmd_lock);
3076         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3077         if (!rc)
3078                 fltr->filter_id = resp->ntuple_filter_id;
3079         mutex_unlock(&bp->hwrm_cmd_lock);
3080         return rc;
3081 }
3082 #endif
3083
3084 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3085                                      u8 *mac_addr)
3086 {
3087         u32 rc = 0;
3088         struct hwrm_cfa_l2_filter_alloc_input req = {0};
3089         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3090
3091         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3092         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX |
3093                                 CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
3094         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
3095         req.enables =
3096                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
3097                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
3098                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3099         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3100         req.l2_addr_mask[0] = 0xff;
3101         req.l2_addr_mask[1] = 0xff;
3102         req.l2_addr_mask[2] = 0xff;
3103         req.l2_addr_mask[3] = 0xff;
3104         req.l2_addr_mask[4] = 0xff;
3105         req.l2_addr_mask[5] = 0xff;
3106
3107         mutex_lock(&bp->hwrm_cmd_lock);
3108         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3109         if (!rc)
3110                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3111                                                         resp->l2_filter_id;
3112         mutex_unlock(&bp->hwrm_cmd_lock);
3113         return rc;
3114 }
3115
3116 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3117 {
3118         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3119         int rc = 0;
3120
3121         /* Any associated ntuple filters will also be cleared by firmware. */
3122         mutex_lock(&bp->hwrm_cmd_lock);
3123         for (i = 0; i < num_of_vnics; i++) {
3124                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3125
3126                 for (j = 0; j < vnic->uc_filter_count; j++) {
3127                         struct hwrm_cfa_l2_filter_free_input req = {0};
3128
3129                         bnxt_hwrm_cmd_hdr_init(bp, &req,
3130                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
3131
3132                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
3133
3134                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3135                                                 HWRM_CMD_TIMEOUT);
3136                 }
3137                 vnic->uc_filter_count = 0;
3138         }
3139         mutex_unlock(&bp->hwrm_cmd_lock);
3140
3141         return rc;
3142 }
3143
3144 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3145 {
3146         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3147         struct hwrm_vnic_tpa_cfg_input req = {0};
3148
3149         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3150
3151         if (tpa_flags) {
3152                 u16 mss = bp->dev->mtu - 40;
3153                 u32 nsegs, n, segs = 0, flags;
3154
3155                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3156                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3157                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3158                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3159                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3160                 if (tpa_flags & BNXT_FLAG_GRO)
3161                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3162
3163                 req.flags = cpu_to_le32(flags);
3164
3165                 req.enables =
3166                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3167                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3168                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3169
3170                 /* Number of segs are log2 units, and first packet is not
3171                  * included as part of this units.
3172                  */
3173                 if (mss <= BNXT_RX_PAGE_SIZE) {
3174                         n = BNXT_RX_PAGE_SIZE / mss;
3175                         nsegs = (MAX_SKB_FRAGS - 1) * n;
3176                 } else {
3177                         n = mss / BNXT_RX_PAGE_SIZE;
3178                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
3179                                 n++;
3180                         nsegs = (MAX_SKB_FRAGS - n) / n;
3181                 }
3182
3183                 segs = ilog2(nsegs);
3184                 req.max_agg_segs = cpu_to_le16(segs);
3185                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3186
3187                 req.min_agg_len = cpu_to_le32(512);
3188         }
3189         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3190
3191         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3192 }
3193
3194 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3195 {
3196         u32 i, j, max_rings;
3197         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3198         struct hwrm_vnic_rss_cfg_input req = {0};
3199
3200         if (vnic->fw_rss_cos_lb_ctx == INVALID_HW_RING_ID)
3201                 return 0;
3202
3203         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3204         if (set_rss) {
3205                 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
3206                                  BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
3207                                  BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
3208                                  BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
3209
3210                 req.hash_type = cpu_to_le32(vnic->hash_type);
3211
3212                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3213                         max_rings = bp->rx_nr_rings;
3214                 else
3215                         max_rings = 1;
3216
3217                 /* Fill the RSS indirection table with ring group ids */
3218                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3219                         if (j == max_rings)
3220                                 j = 0;
3221                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3222                 }
3223
3224                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3225                 req.hash_key_tbl_addr =
3226                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
3227         }
3228         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3229         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3230 }
3231
3232 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3233 {
3234         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3235         struct hwrm_vnic_plcmodes_cfg_input req = {0};
3236
3237         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3238         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3239                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3240                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3241         req.enables =
3242                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3243                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3244         /* thresholds not implemented in firmware yet */
3245         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3246         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3247         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3248         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3249 }
3250
3251 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id)
3252 {
3253         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3254
3255         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3256         req.rss_cos_lb_ctx_id =
3257                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx);
3258
3259         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3260         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3261 }
3262
3263 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3264 {
3265         int i;
3266
3267         for (i = 0; i < bp->nr_vnics; i++) {
3268                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3269
3270                 if (vnic->fw_rss_cos_lb_ctx != INVALID_HW_RING_ID)
3271                         bnxt_hwrm_vnic_ctx_free_one(bp, i);
3272         }
3273         bp->rsscos_nr_ctxs = 0;
3274 }
3275
3276 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id)
3277 {
3278         int rc;
3279         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3280         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3281                                                 bp->hwrm_cmd_resp_addr;
3282
3283         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3284                                -1);
3285
3286         mutex_lock(&bp->hwrm_cmd_lock);
3287         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3288         if (!rc)
3289                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx =
3290                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
3291         mutex_unlock(&bp->hwrm_cmd_lock);
3292
3293         return rc;
3294 }
3295
3296 static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3297 {
3298         unsigned int ring = 0, grp_idx;
3299         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3300         struct hwrm_vnic_cfg_input req = {0};
3301         u16 def_vlan = 0;
3302
3303         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3304         /* Only RSS support for now TBD: COS & LB */
3305         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP |
3306                                   VNIC_CFG_REQ_ENABLES_RSS_RULE);
3307         req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3308         req.cos_rule = cpu_to_le16(0xffff);
3309         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3310                 ring = 0;
3311         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
3312                 ring = vnic_id - 1;
3313
3314         grp_idx = bp->rx_ring[ring].bnapi->index;
3315         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3316         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3317
3318         req.lb_rule = cpu_to_le16(0xffff);
3319         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3320                               VLAN_HLEN);
3321
3322 #ifdef CONFIG_BNXT_SRIOV
3323         if (BNXT_VF(bp))
3324                 def_vlan = bp->vf.vlan;
3325 #endif
3326         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
3327                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
3328
3329         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3330 }
3331
3332 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
3333 {
3334         u32 rc = 0;
3335
3336         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
3337                 struct hwrm_vnic_free_input req = {0};
3338
3339                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
3340                 req.vnic_id =
3341                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
3342
3343                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3344                 if (rc)
3345                         return rc;
3346                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
3347         }
3348         return rc;
3349 }
3350
3351 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
3352 {
3353         u16 i;
3354
3355         for (i = 0; i < bp->nr_vnics; i++)
3356                 bnxt_hwrm_vnic_free_one(bp, i);
3357 }
3358
3359 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
3360                                 unsigned int start_rx_ring_idx,
3361                                 unsigned int nr_rings)
3362 {
3363         int rc = 0;
3364         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
3365         struct hwrm_vnic_alloc_input req = {0};
3366         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3367
3368         /* map ring groups to this vnic */
3369         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
3370                 grp_idx = bp->rx_ring[i].bnapi->index;
3371                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
3372                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
3373                                    j, nr_rings);
3374                         break;
3375                 }
3376                 bp->vnic_info[vnic_id].fw_grp_ids[j] =
3377                                         bp->grp_info[grp_idx].fw_grp_id;
3378         }
3379
3380         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3381         if (vnic_id == 0)
3382                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
3383
3384         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
3385
3386         mutex_lock(&bp->hwrm_cmd_lock);
3387         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3388         if (!rc)
3389                 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
3390         mutex_unlock(&bp->hwrm_cmd_lock);
3391         return rc;
3392 }
3393
3394 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
3395 {
3396         u16 i;
3397         u32 rc = 0;
3398
3399         mutex_lock(&bp->hwrm_cmd_lock);
3400         for (i = 0; i < bp->rx_nr_rings; i++) {
3401                 struct hwrm_ring_grp_alloc_input req = {0};
3402                 struct hwrm_ring_grp_alloc_output *resp =
3403                                         bp->hwrm_cmd_resp_addr;
3404                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
3405
3406                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
3407
3408                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
3409                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
3410                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
3411                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
3412
3413                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3414                                         HWRM_CMD_TIMEOUT);
3415                 if (rc)
3416                         break;
3417
3418                 bp->grp_info[grp_idx].fw_grp_id =
3419                         le32_to_cpu(resp->ring_group_id);
3420         }
3421         mutex_unlock(&bp->hwrm_cmd_lock);
3422         return rc;
3423 }
3424
3425 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
3426 {
3427         u16 i;
3428         u32 rc = 0;
3429         struct hwrm_ring_grp_free_input req = {0};
3430
3431         if (!bp->grp_info)
3432                 return 0;
3433
3434         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
3435
3436         mutex_lock(&bp->hwrm_cmd_lock);
3437         for (i = 0; i < bp->cp_nr_rings; i++) {
3438                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
3439                         continue;
3440                 req.ring_group_id =
3441                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
3442
3443                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3444                                         HWRM_CMD_TIMEOUT);
3445                 if (rc)
3446                         break;
3447                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3448         }
3449         mutex_unlock(&bp->hwrm_cmd_lock);
3450         return rc;
3451 }
3452
3453 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
3454                                     struct bnxt_ring_struct *ring,
3455                                     u32 ring_type, u32 map_index,
3456                                     u32 stats_ctx_id)
3457 {
3458         int rc = 0, err = 0;
3459         struct hwrm_ring_alloc_input req = {0};
3460         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3461         u16 ring_id;
3462
3463         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
3464
3465         req.enables = 0;
3466         if (ring->nr_pages > 1) {
3467                 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
3468                 /* Page size is in log2 units */
3469                 req.page_size = BNXT_PAGE_SHIFT;
3470                 req.page_tbl_depth = 1;
3471         } else {
3472                 req.page_tbl_addr =  cpu_to_le64(ring->dma_arr[0]);
3473         }
3474         req.fbo = 0;
3475         /* Association of ring index with doorbell index and MSIX number */
3476         req.logical_id = cpu_to_le16(map_index);
3477
3478         switch (ring_type) {
3479         case HWRM_RING_ALLOC_TX:
3480                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
3481                 /* Association of transmit ring with completion ring */
3482                 req.cmpl_ring_id =
3483                         cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
3484                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
3485                 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
3486                 req.queue_id = cpu_to_le16(ring->queue_id);
3487                 break;
3488         case HWRM_RING_ALLOC_RX:
3489                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3490                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
3491                 break;
3492         case HWRM_RING_ALLOC_AGG:
3493                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3494                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
3495                 break;
3496         case HWRM_RING_ALLOC_CMPL:
3497                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
3498                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
3499                 if (bp->flags & BNXT_FLAG_USING_MSIX)
3500                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
3501                 break;
3502         default:
3503                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
3504                            ring_type);
3505                 return -1;
3506         }
3507
3508         mutex_lock(&bp->hwrm_cmd_lock);
3509         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3510         err = le16_to_cpu(resp->error_code);
3511         ring_id = le16_to_cpu(resp->ring_id);
3512         mutex_unlock(&bp->hwrm_cmd_lock);
3513
3514         if (rc || err) {
3515                 switch (ring_type) {
3516                 case RING_FREE_REQ_RING_TYPE_CMPL:
3517                         netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
3518                                    rc, err);
3519                         return -1;
3520
3521                 case RING_FREE_REQ_RING_TYPE_RX:
3522                         netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
3523                                    rc, err);
3524                         return -1;
3525
3526                 case RING_FREE_REQ_RING_TYPE_TX:
3527                         netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
3528                                    rc, err);
3529                         return -1;
3530
3531                 default:
3532                         netdev_err(bp->dev, "Invalid ring\n");
3533                         return -1;
3534                 }
3535         }
3536         ring->fw_ring_id = ring_id;
3537         return rc;
3538 }
3539
3540 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
3541 {
3542         int i, rc = 0;
3543
3544         for (i = 0; i < bp->cp_nr_rings; i++) {
3545                 struct bnxt_napi *bnapi = bp->bnapi[i];
3546                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3547                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3548
3549                 cpr->cp_doorbell = bp->bar1 + i * 0x80;
3550                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
3551                                               INVALID_STATS_CTX_ID);
3552                 if (rc)
3553                         goto err_out;
3554                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3555                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
3556         }
3557
3558         for (i = 0; i < bp->tx_nr_rings; i++) {
3559                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3560                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3561                 u32 map_idx = txr->bnapi->index;
3562                 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
3563
3564                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
3565                                               map_idx, fw_stats_ctx);
3566                 if (rc)
3567                         goto err_out;
3568                 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
3569         }
3570
3571         for (i = 0; i < bp->rx_nr_rings; i++) {
3572                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3573                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3574                 u32 map_idx = rxr->bnapi->index;
3575
3576                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
3577                                               map_idx, INVALID_STATS_CTX_ID);
3578                 if (rc)
3579                         goto err_out;
3580                 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
3581                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
3582                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3583         }
3584
3585         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3586                 for (i = 0; i < bp->rx_nr_rings; i++) {
3587                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3588                         struct bnxt_ring_struct *ring =
3589                                                 &rxr->rx_agg_ring_struct;
3590                         u32 grp_idx = rxr->bnapi->index;
3591                         u32 map_idx = grp_idx + bp->rx_nr_rings;
3592
3593                         rc = hwrm_ring_alloc_send_msg(bp, ring,
3594                                                       HWRM_RING_ALLOC_AGG,
3595                                                       map_idx,
3596                                                       INVALID_STATS_CTX_ID);
3597                         if (rc)
3598                                 goto err_out;
3599
3600                         rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
3601                         writel(DB_KEY_RX | rxr->rx_agg_prod,
3602                                rxr->rx_agg_doorbell);
3603                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
3604                 }
3605         }
3606 err_out:
3607         return rc;
3608 }
3609
3610 static int hwrm_ring_free_send_msg(struct bnxt *bp,
3611                                    struct bnxt_ring_struct *ring,
3612                                    u32 ring_type, int cmpl_ring_id)
3613 {
3614         int rc;
3615         struct hwrm_ring_free_input req = {0};
3616         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
3617         u16 error_code;
3618
3619         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
3620         req.ring_type = ring_type;
3621         req.ring_id = cpu_to_le16(ring->fw_ring_id);
3622
3623         mutex_lock(&bp->hwrm_cmd_lock);
3624         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3625         error_code = le16_to_cpu(resp->error_code);
3626         mutex_unlock(&bp->hwrm_cmd_lock);
3627
3628         if (rc || error_code) {
3629                 switch (ring_type) {
3630                 case RING_FREE_REQ_RING_TYPE_CMPL:
3631                         netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
3632                                    rc);
3633                         return rc;
3634                 case RING_FREE_REQ_RING_TYPE_RX:
3635                         netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
3636                                    rc);
3637                         return rc;
3638                 case RING_FREE_REQ_RING_TYPE_TX:
3639                         netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
3640                                    rc);
3641                         return rc;
3642                 default:
3643                         netdev_err(bp->dev, "Invalid ring\n");
3644                         return -1;
3645                 }
3646         }
3647         return 0;
3648 }
3649
3650 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
3651 {
3652         int i;
3653
3654         if (!bp->bnapi)
3655                 return;
3656
3657         for (i = 0; i < bp->tx_nr_rings; i++) {
3658                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3659                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3660                 u32 grp_idx = txr->bnapi->index;
3661                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3662
3663                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3664                         hwrm_ring_free_send_msg(bp, ring,
3665                                                 RING_FREE_REQ_RING_TYPE_TX,
3666                                                 close_path ? cmpl_ring_id :
3667                                                 INVALID_HW_RING_ID);
3668                         ring->fw_ring_id = INVALID_HW_RING_ID;
3669                 }
3670         }
3671
3672         for (i = 0; i < bp->rx_nr_rings; i++) {
3673                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3674                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3675                 u32 grp_idx = rxr->bnapi->index;
3676                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3677
3678                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3679                         hwrm_ring_free_send_msg(bp, ring,
3680                                                 RING_FREE_REQ_RING_TYPE_RX,
3681                                                 close_path ? cmpl_ring_id :
3682                                                 INVALID_HW_RING_ID);
3683                         ring->fw_ring_id = INVALID_HW_RING_ID;
3684                         bp->grp_info[grp_idx].rx_fw_ring_id =
3685                                 INVALID_HW_RING_ID;
3686                 }
3687         }
3688
3689         for (i = 0; i < bp->rx_nr_rings; i++) {
3690                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3691                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
3692                 u32 grp_idx = rxr->bnapi->index;
3693                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3694
3695                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3696                         hwrm_ring_free_send_msg(bp, ring,
3697                                                 RING_FREE_REQ_RING_TYPE_RX,
3698                                                 close_path ? cmpl_ring_id :
3699                                                 INVALID_HW_RING_ID);
3700                         ring->fw_ring_id = INVALID_HW_RING_ID;
3701                         bp->grp_info[grp_idx].agg_fw_ring_id =
3702                                 INVALID_HW_RING_ID;
3703                 }
3704         }
3705
3706         for (i = 0; i < bp->cp_nr_rings; i++) {
3707                 struct bnxt_napi *bnapi = bp->bnapi[i];
3708                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3709                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3710
3711                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3712                         hwrm_ring_free_send_msg(bp, ring,
3713                                                 RING_FREE_REQ_RING_TYPE_CMPL,
3714                                                 INVALID_HW_RING_ID);
3715                         ring->fw_ring_id = INVALID_HW_RING_ID;
3716                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3717                 }
3718         }
3719 }
3720
3721 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
3722         u32 buf_tmrs, u16 flags,
3723         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
3724 {
3725         req->flags = cpu_to_le16(flags);
3726         req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
3727         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
3728         req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
3729         req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
3730         /* Minimum time between 2 interrupts set to buf_tmr x 2 */
3731         req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
3732         req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
3733         req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
3734 }
3735
3736 int bnxt_hwrm_set_coal(struct bnxt *bp)
3737 {
3738         int i, rc = 0;
3739         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
3740                                                            req_tx = {0}, *req;
3741         u16 max_buf, max_buf_irq;
3742         u16 buf_tmr, buf_tmr_irq;
3743         u32 flags;
3744
3745         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
3746                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3747         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
3748                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3749
3750         /* Each rx completion (2 records) should be DMAed immediately.
3751          * DMA 1/4 of the completion buffers at a time.
3752          */
3753         max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
3754         /* max_buf must not be zero */
3755         max_buf = clamp_t(u16, max_buf, 1, 63);
3756         max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
3757         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
3758         /* buf timer set to 1/4 of interrupt timer */
3759         buf_tmr = max_t(u16, buf_tmr / 4, 1);
3760         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
3761         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3762
3763         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3764
3765         /* RING_IDLE generates more IRQs for lower latency.  Enable it only
3766          * if coal_ticks is less than 25 us.
3767          */
3768         if (bp->rx_coal_ticks < 25)
3769                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
3770
3771         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3772                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
3773
3774         /* max_buf must not be zero */
3775         max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
3776         max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
3777         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
3778         /* buf timer set to 1/4 of interrupt timer */
3779         buf_tmr = max_t(u16, buf_tmr / 4, 1);
3780         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
3781         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3782
3783         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3784         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3785                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
3786
3787         mutex_lock(&bp->hwrm_cmd_lock);
3788         for (i = 0; i < bp->cp_nr_rings; i++) {
3789                 struct bnxt_napi *bnapi = bp->bnapi[i];
3790
3791                 req = &req_rx;
3792                 if (!bnapi->rx_ring)
3793                         req = &req_tx;
3794                 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
3795
3796                 rc = _hwrm_send_message(bp, req, sizeof(*req),
3797                                         HWRM_CMD_TIMEOUT);
3798                 if (rc)
3799                         break;
3800         }
3801         mutex_unlock(&bp->hwrm_cmd_lock);
3802         return rc;
3803 }
3804
3805 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
3806 {
3807         int rc = 0, i;
3808         struct hwrm_stat_ctx_free_input req = {0};
3809
3810         if (!bp->bnapi)
3811                 return 0;
3812
3813         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
3814
3815         mutex_lock(&bp->hwrm_cmd_lock);
3816         for (i = 0; i < bp->cp_nr_rings; i++) {
3817                 struct bnxt_napi *bnapi = bp->bnapi[i];
3818                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3819
3820                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
3821                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
3822
3823                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3824                                                 HWRM_CMD_TIMEOUT);
3825                         if (rc)
3826                                 break;
3827
3828                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3829                 }
3830         }
3831         mutex_unlock(&bp->hwrm_cmd_lock);
3832         return rc;
3833 }
3834
3835 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
3836 {
3837         int rc = 0, i;
3838         struct hwrm_stat_ctx_alloc_input req = {0};
3839         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3840
3841         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
3842
3843         req.update_period_ms = cpu_to_le32(1000);
3844
3845         mutex_lock(&bp->hwrm_cmd_lock);
3846         for (i = 0; i < bp->cp_nr_rings; i++) {
3847                 struct bnxt_napi *bnapi = bp->bnapi[i];
3848                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3849
3850                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
3851
3852                 rc = _hwrm_send_message(bp, &req, sizeof(req),
3853                                         HWRM_CMD_TIMEOUT);
3854                 if (rc)
3855                         break;
3856
3857                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
3858
3859                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
3860         }
3861         mutex_unlock(&bp->hwrm_cmd_lock);
3862         return 0;
3863 }
3864
3865 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
3866 {
3867         struct hwrm_func_qcfg_input req = {0};
3868         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
3869         int rc;
3870
3871         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
3872         req.fid = cpu_to_le16(0xffff);
3873         mutex_lock(&bp->hwrm_cmd_lock);
3874         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3875         if (rc)
3876                 goto func_qcfg_exit;
3877
3878 #ifdef CONFIG_BNXT_SRIOV
3879         if (BNXT_VF(bp)) {
3880                 struct bnxt_vf_info *vf = &bp->vf;
3881
3882                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
3883         }
3884 #endif
3885         switch (resp->port_partition_type) {
3886         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
3887         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
3888         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
3889                 bp->port_partition_type = resp->port_partition_type;
3890                 break;
3891         }
3892
3893 func_qcfg_exit:
3894         mutex_unlock(&bp->hwrm_cmd_lock);
3895         return rc;
3896 }
3897
3898 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
3899 {
3900         int rc = 0;
3901         struct hwrm_func_qcaps_input req = {0};
3902         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
3903
3904         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
3905         req.fid = cpu_to_le16(0xffff);
3906
3907         mutex_lock(&bp->hwrm_cmd_lock);
3908         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3909         if (rc)
3910                 goto hwrm_func_qcaps_exit;
3911
3912         if (BNXT_PF(bp)) {
3913                 struct bnxt_pf_info *pf = &bp->pf;
3914
3915                 pf->fw_fid = le16_to_cpu(resp->fid);
3916                 pf->port_id = le16_to_cpu(resp->port_id);
3917                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
3918                 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
3919                 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3920                 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3921                 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3922                 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3923                 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3924                 if (!pf->max_hw_ring_grps)
3925                         pf->max_hw_ring_grps = pf->max_tx_rings;
3926                 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3927                 pf->max_vnics = le16_to_cpu(resp->max_vnics);
3928                 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3929                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
3930                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
3931                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
3932                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
3933                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
3934                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
3935                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
3936                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
3937         } else {
3938 #ifdef CONFIG_BNXT_SRIOV
3939                 struct bnxt_vf_info *vf = &bp->vf;
3940
3941                 vf->fw_fid = le16_to_cpu(resp->fid);
3942                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
3943                 if (is_valid_ether_addr(vf->mac_addr))
3944                         /* overwrite netdev dev_adr with admin VF MAC */
3945                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
3946                 else
3947                         random_ether_addr(bp->dev->dev_addr);
3948
3949                 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3950                 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3951                 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3952                 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3953                 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3954                 if (!vf->max_hw_ring_grps)
3955                         vf->max_hw_ring_grps = vf->max_tx_rings;
3956                 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3957                 vf->max_vnics = le16_to_cpu(resp->max_vnics);
3958                 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3959 #endif
3960         }
3961
3962         bp->tx_push_thresh = 0;
3963         if (resp->flags &
3964             cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
3965                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
3966
3967 hwrm_func_qcaps_exit:
3968         mutex_unlock(&bp->hwrm_cmd_lock);
3969         return rc;
3970 }
3971
3972 static int bnxt_hwrm_func_reset(struct bnxt *bp)
3973 {
3974         struct hwrm_func_reset_input req = {0};
3975
3976         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
3977         req.enables = 0;
3978
3979         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
3980 }
3981
3982 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
3983 {
3984         int rc = 0;
3985         struct hwrm_queue_qportcfg_input req = {0};
3986         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
3987         u8 i, *qptr;
3988
3989         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
3990
3991         mutex_lock(&bp->hwrm_cmd_lock);
3992         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3993         if (rc)
3994                 goto qportcfg_exit;
3995
3996         if (!resp->max_configurable_queues) {
3997                 rc = -EINVAL;
3998                 goto qportcfg_exit;
3999         }
4000         bp->max_tc = resp->max_configurable_queues;
4001         if (bp->max_tc > BNXT_MAX_QUEUE)
4002                 bp->max_tc = BNXT_MAX_QUEUE;
4003
4004         qptr = &resp->queue_id0;
4005         for (i = 0; i < bp->max_tc; i++) {
4006                 bp->q_info[i].queue_id = *qptr++;
4007                 bp->q_info[i].queue_profile = *qptr++;
4008         }
4009
4010 qportcfg_exit:
4011         mutex_unlock(&bp->hwrm_cmd_lock);
4012         return rc;
4013 }
4014
4015 static int bnxt_hwrm_ver_get(struct bnxt *bp)
4016 {
4017         int rc;
4018         struct hwrm_ver_get_input req = {0};
4019         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
4020
4021         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
4022         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
4023         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
4024         req.hwrm_intf_min = HWRM_VERSION_MINOR;
4025         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
4026         mutex_lock(&bp->hwrm_cmd_lock);
4027         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4028         if (rc)
4029                 goto hwrm_ver_get_exit;
4030
4031         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
4032
4033         bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
4034                              resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
4035         if (resp->hwrm_intf_maj < 1) {
4036                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
4037                             resp->hwrm_intf_maj, resp->hwrm_intf_min,
4038                             resp->hwrm_intf_upd);
4039                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
4040         }
4041         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
4042                  resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4043                  resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
4044
4045         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4046         if (!bp->hwrm_cmd_timeout)
4047                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4048
4049         if (resp->hwrm_intf_maj >= 1)
4050                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4051
4052         bp->chip_num = le16_to_cpu(resp->chip_num);
4053
4054 hwrm_ver_get_exit:
4055         mutex_unlock(&bp->hwrm_cmd_lock);
4056         return rc;
4057 }
4058
4059 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4060 {
4061         int rc;
4062         struct bnxt_pf_info *pf = &bp->pf;
4063         struct hwrm_port_qstats_input req = {0};
4064
4065         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4066                 return 0;
4067
4068         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4069         req.port_id = cpu_to_le16(pf->port_id);
4070         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4071         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4072         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4073         return rc;
4074 }
4075
4076 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4077 {
4078         if (bp->vxlan_port_cnt) {
4079                 bnxt_hwrm_tunnel_dst_port_free(
4080                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4081         }
4082         bp->vxlan_port_cnt = 0;
4083         if (bp->nge_port_cnt) {
4084                 bnxt_hwrm_tunnel_dst_port_free(
4085                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4086         }
4087         bp->nge_port_cnt = 0;
4088 }
4089
4090 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4091 {
4092         int rc, i;
4093         u32 tpa_flags = 0;
4094
4095         if (set_tpa)
4096                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4097         for (i = 0; i < bp->nr_vnics; i++) {
4098                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4099                 if (rc) {
4100                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
4101                                    rc, i);
4102                         return rc;
4103                 }
4104         }
4105         return 0;
4106 }
4107
4108 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4109 {
4110         int i;
4111
4112         for (i = 0; i < bp->nr_vnics; i++)
4113                 bnxt_hwrm_vnic_set_rss(bp, i, false);
4114 }
4115
4116 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4117                                     bool irq_re_init)
4118 {
4119         if (bp->vnic_info) {
4120                 bnxt_hwrm_clear_vnic_filter(bp);
4121                 /* clear all RSS setting before free vnic ctx */
4122                 bnxt_hwrm_clear_vnic_rss(bp);
4123                 bnxt_hwrm_vnic_ctx_free(bp);
4124                 /* before free the vnic, undo the vnic tpa settings */
4125                 if (bp->flags & BNXT_FLAG_TPA)
4126                         bnxt_set_tpa(bp, false);
4127                 bnxt_hwrm_vnic_free(bp);
4128         }
4129         bnxt_hwrm_ring_free(bp, close_path);
4130         bnxt_hwrm_ring_grp_free(bp);
4131         if (irq_re_init) {
4132                 bnxt_hwrm_stat_ctx_free(bp);
4133                 bnxt_hwrm_free_tunnel_ports(bp);
4134         }
4135 }
4136
4137 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
4138 {
4139         int rc;
4140
4141         /* allocate context for vnic */
4142         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id);
4143         if (rc) {
4144                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4145                            vnic_id, rc);
4146                 goto vnic_setup_err;
4147         }
4148         bp->rsscos_nr_ctxs++;
4149
4150         /* configure default vnic, ring grp */
4151         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
4152         if (rc) {
4153                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
4154                            vnic_id, rc);
4155                 goto vnic_setup_err;
4156         }
4157
4158         /* Enable RSS hashing on vnic */
4159         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
4160         if (rc) {
4161                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
4162                            vnic_id, rc);
4163                 goto vnic_setup_err;
4164         }
4165
4166         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4167                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
4168                 if (rc) {
4169                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
4170                                    vnic_id, rc);
4171                 }
4172         }
4173
4174 vnic_setup_err:
4175         return rc;
4176 }
4177
4178 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
4179 {
4180 #ifdef CONFIG_RFS_ACCEL
4181         int i, rc = 0;
4182
4183         for (i = 0; i < bp->rx_nr_rings; i++) {
4184                 u16 vnic_id = i + 1;
4185                 u16 ring_id = i;
4186
4187                 if (vnic_id >= bp->nr_vnics)
4188                         break;
4189
4190                 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG;
4191                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
4192                 if (rc) {
4193                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4194                                    vnic_id, rc);
4195                         break;
4196                 }
4197                 rc = bnxt_setup_vnic(bp, vnic_id);
4198                 if (rc)
4199                         break;
4200         }
4201         return rc;
4202 #else
4203         return 0;
4204 #endif
4205 }
4206
4207 static int bnxt_cfg_rx_mode(struct bnxt *);
4208 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
4209
4210 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
4211 {
4212         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
4213         int rc = 0;
4214
4215         if (irq_re_init) {
4216                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
4217                 if (rc) {
4218                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
4219                                    rc);
4220                         goto err_out;
4221                 }
4222         }
4223
4224         rc = bnxt_hwrm_ring_alloc(bp);
4225         if (rc) {
4226                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
4227                 goto err_out;
4228         }
4229
4230         rc = bnxt_hwrm_ring_grp_alloc(bp);
4231         if (rc) {
4232                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
4233                 goto err_out;
4234         }
4235
4236         /* default vnic 0 */
4237         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, bp->rx_nr_rings);
4238         if (rc) {
4239                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
4240                 goto err_out;
4241         }
4242
4243         rc = bnxt_setup_vnic(bp, 0);
4244         if (rc)
4245                 goto err_out;
4246
4247         if (bp->flags & BNXT_FLAG_RFS) {
4248                 rc = bnxt_alloc_rfs_vnics(bp);
4249                 if (rc)
4250                         goto err_out;
4251         }
4252
4253         if (bp->flags & BNXT_FLAG_TPA) {
4254                 rc = bnxt_set_tpa(bp, true);
4255                 if (rc)
4256                         goto err_out;
4257         }
4258
4259         if (BNXT_VF(bp))
4260                 bnxt_update_vf_mac(bp);
4261
4262         /* Filter for default vnic 0 */
4263         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
4264         if (rc) {
4265                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
4266                 goto err_out;
4267         }
4268         vnic->uc_filter_count = 1;
4269
4270         vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
4271
4272         if ((bp->dev->flags & IFF_PROMISC) && BNXT_PF(bp))
4273                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
4274
4275         if (bp->dev->flags & IFF_ALLMULTI) {
4276                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
4277                 vnic->mc_list_count = 0;
4278         } else {
4279                 u32 mask = 0;
4280
4281                 bnxt_mc_list_updated(bp, &mask);
4282                 vnic->rx_mask |= mask;
4283         }
4284
4285         rc = bnxt_cfg_rx_mode(bp);
4286         if (rc)
4287                 goto err_out;
4288
4289         rc = bnxt_hwrm_set_coal(bp);
4290         if (rc)
4291                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
4292                             rc);
4293
4294         if (BNXT_VF(bp)) {
4295                 bnxt_hwrm_func_qcfg(bp);
4296                 netdev_update_features(bp->dev);
4297         }
4298
4299         return 0;
4300
4301 err_out:
4302         bnxt_hwrm_resource_free(bp, 0, true);
4303
4304         return rc;
4305 }
4306
4307 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
4308 {
4309         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
4310         return 0;
4311 }
4312
4313 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
4314 {
4315         bnxt_init_rx_rings(bp);
4316         bnxt_init_tx_rings(bp);
4317         bnxt_init_ring_grps(bp, irq_re_init);
4318         bnxt_init_vnics(bp);
4319
4320         return bnxt_init_chip(bp, irq_re_init);
4321 }
4322
4323 static void bnxt_disable_int(struct bnxt *bp)
4324 {
4325         int i;
4326
4327         if (!bp->bnapi)
4328                 return;
4329
4330         for (i = 0; i < bp->cp_nr_rings; i++) {
4331                 struct bnxt_napi *bnapi = bp->bnapi[i];
4332                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4333
4334                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4335         }
4336 }
4337
4338 static void bnxt_enable_int(struct bnxt *bp)
4339 {
4340         int i;
4341
4342         atomic_set(&bp->intr_sem, 0);
4343         for (i = 0; i < bp->cp_nr_rings; i++) {
4344                 struct bnxt_napi *bnapi = bp->bnapi[i];
4345                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4346
4347                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
4348         }
4349 }
4350
4351 static int bnxt_set_real_num_queues(struct bnxt *bp)
4352 {
4353         int rc;
4354         struct net_device *dev = bp->dev;
4355
4356         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings);
4357         if (rc)
4358                 return rc;
4359
4360         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
4361         if (rc)
4362                 return rc;
4363
4364 #ifdef CONFIG_RFS_ACCEL
4365         if (bp->flags & BNXT_FLAG_RFS)
4366                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
4367 #endif
4368
4369         return rc;
4370 }
4371
4372 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4373                            bool shared)
4374 {
4375         int _rx = *rx, _tx = *tx;
4376
4377         if (shared) {
4378                 *rx = min_t(int, _rx, max);
4379                 *tx = min_t(int, _tx, max);
4380         } else {
4381                 if (max < 2)
4382                         return -ENOMEM;
4383
4384                 while (_rx + _tx > max) {
4385                         if (_rx > _tx && _rx > 1)
4386                                 _rx--;
4387                         else if (_tx > 1)
4388                                 _tx--;
4389                 }
4390                 *rx = _rx;
4391                 *tx = _tx;
4392         }
4393         return 0;
4394 }
4395
4396 static int bnxt_setup_msix(struct bnxt *bp)
4397 {
4398         struct msix_entry *msix_ent;
4399         struct net_device *dev = bp->dev;
4400         int i, total_vecs, rc = 0, min = 1;
4401         const int len = sizeof(bp->irq_tbl[0].name);
4402
4403         bp->flags &= ~BNXT_FLAG_USING_MSIX;
4404         total_vecs = bp->cp_nr_rings;
4405
4406         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
4407         if (!msix_ent)
4408                 return -ENOMEM;
4409
4410         for (i = 0; i < total_vecs; i++) {
4411                 msix_ent[i].entry = i;
4412                 msix_ent[i].vector = 0;
4413         }
4414
4415         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
4416                 min = 2;
4417
4418         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
4419         if (total_vecs < 0) {
4420                 rc = -ENODEV;
4421                 goto msix_setup_exit;
4422         }
4423
4424         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
4425         if (bp->irq_tbl) {
4426                 int tcs;
4427
4428                 /* Trim rings based upon num of vectors allocated */
4429                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
4430                                      total_vecs, min == 1);
4431                 if (rc)
4432                         goto msix_setup_exit;
4433
4434                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4435                 tcs = netdev_get_num_tc(dev);
4436                 if (tcs > 1) {
4437                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs;
4438                         if (bp->tx_nr_rings_per_tc == 0) {
4439                                 netdev_reset_tc(dev);
4440                                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4441                         } else {
4442                                 int i, off, count;
4443
4444                                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs;
4445                                 for (i = 0; i < tcs; i++) {
4446                                         count = bp->tx_nr_rings_per_tc;
4447                                         off = i * count;
4448                                         netdev_set_tc_queue(dev, i, count, off);
4449                                 }
4450                         }
4451                 }
4452                 bp->cp_nr_rings = total_vecs;
4453
4454                 for (i = 0; i < bp->cp_nr_rings; i++) {
4455                         char *attr;
4456
4457                         bp->irq_tbl[i].vector = msix_ent[i].vector;
4458                         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4459                                 attr = "TxRx";
4460                         else if (i < bp->rx_nr_rings)
4461                                 attr = "rx";
4462                         else
4463                                 attr = "tx";
4464
4465                         snprintf(bp->irq_tbl[i].name, len,
4466                                  "%s-%s-%d", dev->name, attr, i);
4467                         bp->irq_tbl[i].handler = bnxt_msix;
4468                 }
4469                 rc = bnxt_set_real_num_queues(bp);
4470                 if (rc)
4471                         goto msix_setup_exit;
4472         } else {
4473                 rc = -ENOMEM;
4474                 goto msix_setup_exit;
4475         }
4476         bp->flags |= BNXT_FLAG_USING_MSIX;
4477         kfree(msix_ent);
4478         return 0;
4479
4480 msix_setup_exit:
4481         netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc);
4482         pci_disable_msix(bp->pdev);
4483         kfree(msix_ent);
4484         return rc;
4485 }
4486
4487 static int bnxt_setup_inta(struct bnxt *bp)
4488 {
4489         int rc;
4490         const int len = sizeof(bp->irq_tbl[0].name);
4491
4492         if (netdev_get_num_tc(bp->dev))
4493                 netdev_reset_tc(bp->dev);
4494
4495         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
4496         if (!bp->irq_tbl) {
4497                 rc = -ENOMEM;
4498                 return rc;
4499         }
4500         bp->rx_nr_rings = 1;
4501         bp->tx_nr_rings = 1;
4502         bp->cp_nr_rings = 1;
4503         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4504         bp->flags |= BNXT_FLAG_SHARED_RINGS;
4505         bp->irq_tbl[0].vector = bp->pdev->irq;
4506         snprintf(bp->irq_tbl[0].name, len,
4507                  "%s-%s-%d", bp->dev->name, "TxRx", 0);
4508         bp->irq_tbl[0].handler = bnxt_inta;
4509         rc = bnxt_set_real_num_queues(bp);
4510         return rc;
4511 }
4512
4513 static int bnxt_setup_int_mode(struct bnxt *bp)
4514 {
4515         int rc = 0;
4516
4517         if (bp->flags & BNXT_FLAG_MSIX_CAP)
4518                 rc = bnxt_setup_msix(bp);
4519
4520         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
4521                 /* fallback to INTA */
4522                 rc = bnxt_setup_inta(bp);
4523         }
4524         return rc;
4525 }
4526
4527 static void bnxt_free_irq(struct bnxt *bp)
4528 {
4529         struct bnxt_irq *irq;
4530         int i;
4531
4532 #ifdef CONFIG_RFS_ACCEL
4533         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
4534         bp->dev->rx_cpu_rmap = NULL;
4535 #endif
4536         if (!bp->irq_tbl)
4537                 return;
4538
4539         for (i = 0; i < bp->cp_nr_rings; i++) {
4540                 irq = &bp->irq_tbl[i];
4541                 if (irq->requested)
4542                         free_irq(irq->vector, bp->bnapi[i]);
4543                 irq->requested = 0;
4544         }
4545         if (bp->flags & BNXT_FLAG_USING_MSIX)
4546                 pci_disable_msix(bp->pdev);
4547         kfree(bp->irq_tbl);
4548         bp->irq_tbl = NULL;
4549 }
4550
4551 static int bnxt_request_irq(struct bnxt *bp)
4552 {
4553         int i, j, rc = 0;
4554         unsigned long flags = 0;
4555 #ifdef CONFIG_RFS_ACCEL
4556         struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
4557 #endif
4558
4559         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
4560                 flags = IRQF_SHARED;
4561
4562         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
4563                 struct bnxt_irq *irq = &bp->irq_tbl[i];
4564 #ifdef CONFIG_RFS_ACCEL
4565                 if (rmap && bp->bnapi[i]->rx_ring) {
4566                         rc = irq_cpu_rmap_add(rmap, irq->vector);
4567                         if (rc)
4568                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
4569                                             j);
4570                         j++;
4571                 }
4572 #endif
4573                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
4574                                  bp->bnapi[i]);
4575                 if (rc)
4576                         break;
4577
4578                 irq->requested = 1;
4579         }
4580         return rc;
4581 }
4582
4583 static void bnxt_del_napi(struct bnxt *bp)
4584 {
4585         int i;
4586
4587         if (!bp->bnapi)
4588                 return;
4589
4590         for (i = 0; i < bp->cp_nr_rings; i++) {
4591                 struct bnxt_napi *bnapi = bp->bnapi[i];
4592
4593                 napi_hash_del(&bnapi->napi);
4594                 netif_napi_del(&bnapi->napi);
4595         }
4596 }
4597
4598 static void bnxt_init_napi(struct bnxt *bp)
4599 {
4600         int i;
4601         struct bnxt_napi *bnapi;
4602
4603         if (bp->flags & BNXT_FLAG_USING_MSIX) {
4604                 for (i = 0; i < bp->cp_nr_rings; i++) {
4605                         bnapi = bp->bnapi[i];
4606                         netif_napi_add(bp->dev, &bnapi->napi,
4607                                        bnxt_poll, 64);
4608                 }
4609         } else {
4610                 bnapi = bp->bnapi[0];
4611                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
4612         }
4613 }
4614
4615 static void bnxt_disable_napi(struct bnxt *bp)
4616 {
4617         int i;
4618
4619         if (!bp->bnapi)
4620                 return;
4621
4622         for (i = 0; i < bp->cp_nr_rings; i++) {
4623                 napi_disable(&bp->bnapi[i]->napi);
4624                 bnxt_disable_poll(bp->bnapi[i]);
4625         }
4626 }
4627
4628 static void bnxt_enable_napi(struct bnxt *bp)
4629 {
4630         int i;
4631
4632         for (i = 0; i < bp->cp_nr_rings; i++) {
4633                 bp->bnapi[i]->in_reset = false;
4634                 bnxt_enable_poll(bp->bnapi[i]);
4635                 napi_enable(&bp->bnapi[i]->napi);
4636         }
4637 }
4638
4639 static void bnxt_tx_disable(struct bnxt *bp)
4640 {
4641         int i;
4642         struct bnxt_tx_ring_info *txr;
4643         struct netdev_queue *txq;
4644
4645         if (bp->tx_ring) {
4646                 for (i = 0; i < bp->tx_nr_rings; i++) {
4647                         txr = &bp->tx_ring[i];
4648                         txq = netdev_get_tx_queue(bp->dev, i);
4649                         __netif_tx_lock(txq, smp_processor_id());
4650                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
4651                         __netif_tx_unlock(txq);
4652                 }
4653         }
4654         /* Stop all TX queues */
4655         netif_tx_disable(bp->dev);
4656         netif_carrier_off(bp->dev);
4657 }
4658
4659 static void bnxt_tx_enable(struct bnxt *bp)
4660 {
4661         int i;
4662         struct bnxt_tx_ring_info *txr;
4663         struct netdev_queue *txq;
4664
4665         for (i = 0; i < bp->tx_nr_rings; i++) {
4666                 txr = &bp->tx_ring[i];
4667                 txq = netdev_get_tx_queue(bp->dev, i);
4668                 txr->dev_state = 0;
4669         }
4670         netif_tx_wake_all_queues(bp->dev);
4671         if (bp->link_info.link_up)
4672                 netif_carrier_on(bp->dev);
4673 }
4674
4675 static void bnxt_report_link(struct bnxt *bp)
4676 {
4677         if (bp->link_info.link_up) {
4678                 const char *duplex;
4679                 const char *flow_ctrl;
4680                 u16 speed;
4681
4682                 netif_carrier_on(bp->dev);
4683                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
4684                         duplex = "full";
4685                 else
4686                         duplex = "half";
4687                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
4688                         flow_ctrl = "ON - receive & transmit";
4689                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
4690                         flow_ctrl = "ON - transmit";
4691                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
4692                         flow_ctrl = "ON - receive";
4693                 else
4694                         flow_ctrl = "none";
4695                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
4696                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
4697                             speed, duplex, flow_ctrl);
4698                 if (bp->flags & BNXT_FLAG_EEE_CAP)
4699                         netdev_info(bp->dev, "EEE is %s\n",
4700                                     bp->eee.eee_active ? "active" :
4701                                                          "not active");
4702         } else {
4703                 netif_carrier_off(bp->dev);
4704                 netdev_err(bp->dev, "NIC Link is Down\n");
4705         }
4706 }
4707
4708 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
4709 {
4710         int rc = 0;
4711         struct hwrm_port_phy_qcaps_input req = {0};
4712         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4713
4714         if (bp->hwrm_spec_code < 0x10201)
4715                 return 0;
4716
4717         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
4718
4719         mutex_lock(&bp->hwrm_cmd_lock);
4720         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4721         if (rc)
4722                 goto hwrm_phy_qcaps_exit;
4723
4724         if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
4725                 struct ethtool_eee *eee = &bp->eee;
4726                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
4727
4728                 bp->flags |= BNXT_FLAG_EEE_CAP;
4729                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4730                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
4731                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
4732                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
4733                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
4734         }
4735
4736 hwrm_phy_qcaps_exit:
4737         mutex_unlock(&bp->hwrm_cmd_lock);
4738         return rc;
4739 }
4740
4741 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
4742 {
4743         int rc = 0;
4744         struct bnxt_link_info *link_info = &bp->link_info;
4745         struct hwrm_port_phy_qcfg_input req = {0};
4746         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4747         u8 link_up = link_info->link_up;
4748
4749         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
4750
4751         mutex_lock(&bp->hwrm_cmd_lock);
4752         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4753         if (rc) {
4754                 mutex_unlock(&bp->hwrm_cmd_lock);
4755                 return rc;
4756         }
4757
4758         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
4759         link_info->phy_link_status = resp->link;
4760         link_info->duplex =  resp->duplex;
4761         link_info->pause = resp->pause;
4762         link_info->auto_mode = resp->auto_mode;
4763         link_info->auto_pause_setting = resp->auto_pause;
4764         link_info->lp_pause = resp->link_partner_adv_pause;
4765         link_info->force_pause_setting = resp->force_pause;
4766         link_info->duplex_setting = resp->duplex;
4767         if (link_info->phy_link_status == BNXT_LINK_LINK)
4768                 link_info->link_speed = le16_to_cpu(resp->link_speed);
4769         else
4770                 link_info->link_speed = 0;
4771         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
4772         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
4773         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
4774         link_info->lp_auto_link_speeds =
4775                 le16_to_cpu(resp->link_partner_adv_speeds);
4776         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
4777         link_info->phy_ver[0] = resp->phy_maj;
4778         link_info->phy_ver[1] = resp->phy_min;
4779         link_info->phy_ver[2] = resp->phy_bld;
4780         link_info->media_type = resp->media_type;
4781         link_info->phy_type = resp->phy_type;
4782         link_info->transceiver = resp->xcvr_pkg_type;
4783         link_info->phy_addr = resp->eee_config_phy_addr &
4784                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
4785         link_info->module_status = resp->module_status;
4786
4787         if (bp->flags & BNXT_FLAG_EEE_CAP) {
4788                 struct ethtool_eee *eee = &bp->eee;
4789                 u16 fw_speeds;
4790
4791                 eee->eee_active = 0;
4792                 if (resp->eee_config_phy_addr &
4793                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
4794                         eee->eee_active = 1;
4795                         fw_speeds = le16_to_cpu(
4796                                 resp->link_partner_adv_eee_link_speed_mask);
4797                         eee->lp_advertised =
4798                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4799                 }
4800
4801                 /* Pull initial EEE config */
4802                 if (!chng_link_state) {
4803                         if (resp->eee_config_phy_addr &
4804                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
4805                                 eee->eee_enabled = 1;
4806
4807                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
4808                         eee->advertised =
4809                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4810
4811                         if (resp->eee_config_phy_addr &
4812                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
4813                                 __le32 tmr;
4814
4815                                 eee->tx_lpi_enabled = 1;
4816                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
4817                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
4818                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
4819                         }
4820                 }
4821         }
4822         /* TODO: need to add more logic to report VF link */
4823         if (chng_link_state) {
4824                 if (link_info->phy_link_status == BNXT_LINK_LINK)
4825                         link_info->link_up = 1;
4826                 else
4827                         link_info->link_up = 0;
4828                 if (link_up != link_info->link_up)
4829                         bnxt_report_link(bp);
4830         } else {
4831                 /* alwasy link down if not require to update link state */
4832                 link_info->link_up = 0;
4833         }
4834         mutex_unlock(&bp->hwrm_cmd_lock);
4835         return 0;
4836 }
4837
4838 static void bnxt_get_port_module_status(struct bnxt *bp)
4839 {
4840         struct bnxt_link_info *link_info = &bp->link_info;
4841         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
4842         u8 module_status;
4843
4844         if (bnxt_update_link(bp, true))
4845                 return;
4846
4847         module_status = link_info->module_status;
4848         switch (module_status) {
4849         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
4850         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
4851         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
4852                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
4853                             bp->pf.port_id);
4854                 if (bp->hwrm_spec_code >= 0x10201) {
4855                         netdev_warn(bp->dev, "Module part number %s\n",
4856                                     resp->phy_vendor_partnumber);
4857                 }
4858                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
4859                         netdev_warn(bp->dev, "TX is disabled\n");
4860                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
4861                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
4862         }
4863 }
4864
4865 static void
4866 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
4867 {
4868         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
4869                 if (bp->hwrm_spec_code >= 0x10201)
4870                         req->auto_pause =
4871                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
4872                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4873                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
4874                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4875                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
4876                 req->enables |=
4877                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4878         } else {
4879                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4880                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
4881                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4882                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
4883                 req->enables |=
4884                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
4885                 if (bp->hwrm_spec_code >= 0x10201) {
4886                         req->auto_pause = req->force_pause;
4887                         req->enables |= cpu_to_le32(
4888                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4889                 }
4890         }
4891 }
4892
4893 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
4894                                       struct hwrm_port_phy_cfg_input *req)
4895 {
4896         u8 autoneg = bp->link_info.autoneg;
4897         u16 fw_link_speed = bp->link_info.req_link_speed;
4898         u32 advertising = bp->link_info.advertising;
4899
4900         if (autoneg & BNXT_AUTONEG_SPEED) {
4901                 req->auto_mode |=
4902                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
4903
4904                 req->enables |= cpu_to_le32(
4905                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
4906                 req->auto_link_speed_mask = cpu_to_le16(advertising);
4907
4908                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
4909                 req->flags |=
4910                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
4911         } else {
4912                 req->force_link_speed = cpu_to_le16(fw_link_speed);
4913                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
4914         }
4915
4916         /* tell chimp that the setting takes effect immediately */
4917         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
4918 }
4919
4920 int bnxt_hwrm_set_pause(struct bnxt *bp)
4921 {
4922         struct hwrm_port_phy_cfg_input req = {0};
4923         int rc;
4924
4925         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4926         bnxt_hwrm_set_pause_common(bp, &req);
4927
4928         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
4929             bp->link_info.force_link_chng)
4930                 bnxt_hwrm_set_link_common(bp, &req);
4931
4932         mutex_lock(&bp->hwrm_cmd_lock);
4933         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4934         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
4935                 /* since changing of pause setting doesn't trigger any link
4936                  * change event, the driver needs to update the current pause
4937                  * result upon successfully return of the phy_cfg command
4938                  */
4939                 bp->link_info.pause =
4940                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
4941                 bp->link_info.auto_pause_setting = 0;
4942                 if (!bp->link_info.force_link_chng)
4943                         bnxt_report_link(bp);
4944         }
4945         bp->link_info.force_link_chng = false;
4946         mutex_unlock(&bp->hwrm_cmd_lock);
4947         return rc;
4948 }
4949
4950 static void bnxt_hwrm_set_eee(struct bnxt *bp,
4951                               struct hwrm_port_phy_cfg_input *req)
4952 {
4953         struct ethtool_eee *eee = &bp->eee;
4954
4955         if (eee->eee_enabled) {
4956                 u16 eee_speeds;
4957                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
4958
4959                 if (eee->tx_lpi_enabled)
4960                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
4961                 else
4962                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
4963
4964                 req->flags |= cpu_to_le32(flags);
4965                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
4966                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
4967                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
4968         } else {
4969                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
4970         }
4971 }
4972
4973 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
4974 {
4975         struct hwrm_port_phy_cfg_input req = {0};
4976
4977         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4978         if (set_pause)
4979                 bnxt_hwrm_set_pause_common(bp, &req);
4980
4981         bnxt_hwrm_set_link_common(bp, &req);
4982
4983         if (set_eee)
4984                 bnxt_hwrm_set_eee(bp, &req);
4985         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4986 }
4987
4988 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
4989 {
4990         struct hwrm_port_phy_cfg_input req = {0};
4991
4992         if (!BNXT_SINGLE_PF(bp))
4993                 return 0;
4994
4995         if (pci_num_vf(bp->pdev))
4996                 return 0;
4997
4998         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4999         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DOWN);
5000         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5001 }
5002
5003 static bool bnxt_eee_config_ok(struct bnxt *bp)
5004 {
5005         struct ethtool_eee *eee = &bp->eee;
5006         struct bnxt_link_info *link_info = &bp->link_info;
5007
5008         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
5009                 return true;
5010
5011         if (eee->eee_enabled) {
5012                 u32 advertising =
5013                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
5014
5015                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
5016                         eee->eee_enabled = 0;
5017                         return false;
5018                 }
5019                 if (eee->advertised & ~advertising) {
5020                         eee->advertised = advertising & eee->supported;
5021                         return false;
5022                 }
5023         }
5024         return true;
5025 }
5026
5027 static int bnxt_update_phy_setting(struct bnxt *bp)
5028 {
5029         int rc;
5030         bool update_link = false;
5031         bool update_pause = false;
5032         bool update_eee = false;
5033         struct bnxt_link_info *link_info = &bp->link_info;
5034
5035         rc = bnxt_update_link(bp, true);
5036         if (rc) {
5037                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
5038                            rc);
5039                 return rc;
5040         }
5041         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
5042             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
5043             link_info->req_flow_ctrl)
5044                 update_pause = true;
5045         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
5046             link_info->force_pause_setting != link_info->req_flow_ctrl)
5047                 update_pause = true;
5048         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
5049                 if (BNXT_AUTO_MODE(link_info->auto_mode))
5050                         update_link = true;
5051                 if (link_info->req_link_speed != link_info->force_link_speed)
5052                         update_link = true;
5053                 if (link_info->req_duplex != link_info->duplex_setting)
5054                         update_link = true;
5055         } else {
5056                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
5057                         update_link = true;
5058                 if (link_info->advertising != link_info->auto_link_speeds)
5059                         update_link = true;
5060         }
5061
5062         if (!bnxt_eee_config_ok(bp))
5063                 update_eee = true;
5064
5065         if (update_link)
5066                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
5067         else if (update_pause)
5068                 rc = bnxt_hwrm_set_pause(bp);
5069         if (rc) {
5070                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
5071                            rc);
5072                 return rc;
5073         }
5074
5075         return rc;
5076 }
5077
5078 /* Common routine to pre-map certain register block to different GRC window.
5079  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
5080  * in PF and 3 windows in VF that can be customized to map in different
5081  * register blocks.
5082  */
5083 static void bnxt_preset_reg_win(struct bnxt *bp)
5084 {
5085         if (BNXT_PF(bp)) {
5086                 /* CAG registers map to GRC window #4 */
5087                 writel(BNXT_CAG_REG_BASE,
5088                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
5089         }
5090 }
5091
5092 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5093 {
5094         int rc = 0;
5095
5096         bnxt_preset_reg_win(bp);
5097         netif_carrier_off(bp->dev);
5098         if (irq_re_init) {
5099                 rc = bnxt_setup_int_mode(bp);
5100                 if (rc) {
5101                         netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
5102                                    rc);
5103                         return rc;
5104                 }
5105         }
5106         if ((bp->flags & BNXT_FLAG_RFS) &&
5107             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
5108                 /* disable RFS if falling back to INTA */
5109                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
5110                 bp->flags &= ~BNXT_FLAG_RFS;
5111         }
5112
5113         rc = bnxt_alloc_mem(bp, irq_re_init);
5114         if (rc) {
5115                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
5116                 goto open_err_free_mem;
5117         }
5118
5119         if (irq_re_init) {
5120                 bnxt_init_napi(bp);
5121                 rc = bnxt_request_irq(bp);
5122                 if (rc) {
5123                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
5124                         goto open_err;
5125                 }
5126         }
5127
5128         bnxt_enable_napi(bp);
5129
5130         rc = bnxt_init_nic(bp, irq_re_init);
5131         if (rc) {
5132                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
5133                 goto open_err;
5134         }
5135
5136         if (link_re_init) {
5137                 rc = bnxt_update_phy_setting(bp);
5138                 if (rc)
5139                         netdev_warn(bp->dev, "failed to update phy settings\n");
5140         }
5141
5142         if (irq_re_init) {
5143 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
5144                 vxlan_get_rx_port(bp->dev);
5145 #endif
5146                 if (!bnxt_hwrm_tunnel_dst_port_alloc(
5147                                 bp, htons(0x17c1),
5148                                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
5149                         bp->nge_port_cnt = 1;
5150         }
5151
5152         set_bit(BNXT_STATE_OPEN, &bp->state);
5153         bnxt_enable_int(bp);
5154         /* Enable TX queues */
5155         bnxt_tx_enable(bp);
5156         mod_timer(&bp->timer, jiffies + bp->current_interval);
5157         /* Poll link status and check for SFP+ module status */
5158         bnxt_get_port_module_status(bp);
5159
5160         return 0;
5161
5162 open_err:
5163         bnxt_disable_napi(bp);
5164         bnxt_del_napi(bp);
5165
5166 open_err_free_mem:
5167         bnxt_free_skbs(bp);
5168         bnxt_free_irq(bp);
5169         bnxt_free_mem(bp, true);
5170         return rc;
5171 }
5172
5173 /* rtnl_lock held */
5174 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5175 {
5176         int rc = 0;
5177
5178         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
5179         if (rc) {
5180                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
5181                 dev_close(bp->dev);
5182         }
5183         return rc;
5184 }
5185
5186 static int bnxt_open(struct net_device *dev)
5187 {
5188         struct bnxt *bp = netdev_priv(dev);
5189         int rc = 0;
5190
5191         rc = bnxt_hwrm_func_reset(bp);
5192         if (rc) {
5193                 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n",
5194                            rc);
5195                 rc = -1;
5196                 return rc;
5197         }
5198         return __bnxt_open_nic(bp, true, true);
5199 }
5200
5201 static void bnxt_disable_int_sync(struct bnxt *bp)
5202 {
5203         int i;
5204
5205         atomic_inc(&bp->intr_sem);
5206         if (!netif_running(bp->dev))
5207                 return;
5208
5209         bnxt_disable_int(bp);
5210         for (i = 0; i < bp->cp_nr_rings; i++)
5211                 synchronize_irq(bp->irq_tbl[i].vector);
5212 }
5213
5214 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5215 {
5216         int rc = 0;
5217
5218 #ifdef CONFIG_BNXT_SRIOV
5219         if (bp->sriov_cfg) {
5220                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
5221                                                       !bp->sriov_cfg,
5222                                                       BNXT_SRIOV_CFG_WAIT_TMO);
5223                 if (rc)
5224                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
5225         }
5226 #endif
5227         /* Change device state to avoid TX queue wake up's */
5228         bnxt_tx_disable(bp);
5229
5230         clear_bit(BNXT_STATE_OPEN, &bp->state);
5231         smp_mb__after_atomic();
5232         while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state))
5233                 msleep(20);
5234
5235         /* Flush rings before disabling interrupts */
5236         bnxt_shutdown_nic(bp, irq_re_init);
5237
5238         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
5239
5240         bnxt_disable_napi(bp);
5241         bnxt_disable_int_sync(bp);
5242         del_timer_sync(&bp->timer);
5243         bnxt_free_skbs(bp);
5244
5245         if (irq_re_init) {
5246                 bnxt_free_irq(bp);
5247                 bnxt_del_napi(bp);
5248         }
5249         bnxt_free_mem(bp, irq_re_init);
5250         return rc;
5251 }
5252
5253 static int bnxt_close(struct net_device *dev)
5254 {
5255         struct bnxt *bp = netdev_priv(dev);
5256
5257         bnxt_close_nic(bp, true, true);
5258         bnxt_hwrm_shutdown_link(bp);
5259         return 0;
5260 }
5261
5262 /* rtnl_lock held */
5263 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5264 {
5265         switch (cmd) {
5266         case SIOCGMIIPHY:
5267                 /* fallthru */
5268         case SIOCGMIIREG: {
5269                 if (!netif_running(dev))
5270                         return -EAGAIN;
5271
5272                 return 0;
5273         }
5274
5275         case SIOCSMIIREG:
5276                 if (!netif_running(dev))
5277                         return -EAGAIN;
5278
5279                 return 0;
5280
5281         default:
5282                 /* do nothing */
5283                 break;
5284         }
5285         return -EOPNOTSUPP;
5286 }
5287
5288 static struct rtnl_link_stats64 *
5289 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
5290 {
5291         u32 i;
5292         struct bnxt *bp = netdev_priv(dev);
5293
5294         memset(stats, 0, sizeof(struct rtnl_link_stats64));
5295
5296         if (!bp->bnapi)
5297                 return stats;
5298
5299         /* TODO check if we need to synchronize with bnxt_close path */
5300         for (i = 0; i < bp->cp_nr_rings; i++) {
5301                 struct bnxt_napi *bnapi = bp->bnapi[i];
5302                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5303                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
5304
5305                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
5306                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
5307                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
5308
5309                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
5310                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
5311                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
5312
5313                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
5314                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
5315                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
5316
5317                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
5318                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
5319                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
5320
5321                 stats->rx_missed_errors +=
5322                         le64_to_cpu(hw_stats->rx_discard_pkts);
5323
5324                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
5325
5326                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
5327         }
5328
5329         if (bp->flags & BNXT_FLAG_PORT_STATS) {
5330                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
5331                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
5332
5333                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
5334                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
5335                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
5336                                           le64_to_cpu(rx->rx_ovrsz_frames) +
5337                                           le64_to_cpu(rx->rx_runt_frames);
5338                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
5339                                    le64_to_cpu(rx->rx_jbr_frames);
5340                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
5341                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
5342                 stats->tx_errors = le64_to_cpu(tx->tx_err);
5343         }
5344
5345         return stats;
5346 }
5347
5348 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
5349 {
5350         struct net_device *dev = bp->dev;
5351         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5352         struct netdev_hw_addr *ha;
5353         u8 *haddr;
5354         int mc_count = 0;
5355         bool update = false;
5356         int off = 0;
5357
5358         netdev_for_each_mc_addr(ha, dev) {
5359                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
5360                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5361                         vnic->mc_list_count = 0;
5362                         return false;
5363                 }
5364                 haddr = ha->addr;
5365                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
5366                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
5367                         update = true;
5368                 }
5369                 off += ETH_ALEN;
5370                 mc_count++;
5371         }
5372         if (mc_count)
5373                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
5374
5375         if (mc_count != vnic->mc_list_count) {
5376                 vnic->mc_list_count = mc_count;
5377                 update = true;
5378         }
5379         return update;
5380 }
5381
5382 static bool bnxt_uc_list_updated(struct bnxt *bp)
5383 {
5384         struct net_device *dev = bp->dev;
5385         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5386         struct netdev_hw_addr *ha;
5387         int off = 0;
5388
5389         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
5390                 return true;
5391
5392         netdev_for_each_uc_addr(ha, dev) {
5393                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
5394                         return true;
5395
5396                 off += ETH_ALEN;
5397         }
5398         return false;
5399 }
5400
5401 static void bnxt_set_rx_mode(struct net_device *dev)
5402 {
5403         struct bnxt *bp = netdev_priv(dev);
5404         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5405         u32 mask = vnic->rx_mask;
5406         bool mc_update = false;
5407         bool uc_update;
5408
5409         if (!netif_running(dev))
5410                 return;
5411
5412         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
5413                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
5414                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
5415
5416         /* Only allow PF to be in promiscuous mode */
5417         if ((dev->flags & IFF_PROMISC) && BNXT_PF(bp))
5418                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5419
5420         uc_update = bnxt_uc_list_updated(bp);
5421
5422         if (dev->flags & IFF_ALLMULTI) {
5423                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5424                 vnic->mc_list_count = 0;
5425         } else {
5426                 mc_update = bnxt_mc_list_updated(bp, &mask);
5427         }
5428
5429         if (mask != vnic->rx_mask || uc_update || mc_update) {
5430                 vnic->rx_mask = mask;
5431
5432                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
5433                 schedule_work(&bp->sp_task);
5434         }
5435 }
5436
5437 static int bnxt_cfg_rx_mode(struct bnxt *bp)
5438 {
5439         struct net_device *dev = bp->dev;
5440         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5441         struct netdev_hw_addr *ha;
5442         int i, off = 0, rc;
5443         bool uc_update;
5444
5445         netif_addr_lock_bh(dev);
5446         uc_update = bnxt_uc_list_updated(bp);
5447         netif_addr_unlock_bh(dev);
5448
5449         if (!uc_update)
5450                 goto skip_uc;
5451
5452         mutex_lock(&bp->hwrm_cmd_lock);
5453         for (i = 1; i < vnic->uc_filter_count; i++) {
5454                 struct hwrm_cfa_l2_filter_free_input req = {0};
5455
5456                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
5457                                        -1);
5458
5459                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
5460
5461                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5462                                         HWRM_CMD_TIMEOUT);
5463         }
5464         mutex_unlock(&bp->hwrm_cmd_lock);
5465
5466         vnic->uc_filter_count = 1;
5467
5468         netif_addr_lock_bh(dev);
5469         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
5470                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5471         } else {
5472                 netdev_for_each_uc_addr(ha, dev) {
5473                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
5474                         off += ETH_ALEN;
5475                         vnic->uc_filter_count++;
5476                 }
5477         }
5478         netif_addr_unlock_bh(dev);
5479
5480         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
5481                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
5482                 if (rc) {
5483                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
5484                                    rc);
5485                         vnic->uc_filter_count = i;
5486                         return rc;
5487                 }
5488         }
5489
5490 skip_uc:
5491         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
5492         if (rc)
5493                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
5494                            rc);
5495
5496         return rc;
5497 }
5498
5499 static bool bnxt_rfs_capable(struct bnxt *bp)
5500 {
5501 #ifdef CONFIG_RFS_ACCEL
5502         struct bnxt_pf_info *pf = &bp->pf;
5503         int vnics;
5504
5505         if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP))
5506                 return false;
5507
5508         vnics = 1 + bp->rx_nr_rings;
5509         if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics)
5510                 return false;
5511
5512         return true;
5513 #else
5514         return false;
5515 #endif
5516 }
5517
5518 static netdev_features_t bnxt_fix_features(struct net_device *dev,
5519                                            netdev_features_t features)
5520 {
5521         struct bnxt *bp = netdev_priv(dev);
5522
5523         if (!bnxt_rfs_capable(bp))
5524                 features &= ~NETIF_F_NTUPLE;
5525
5526         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
5527          * turned on or off together.
5528          */
5529         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
5530             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
5531                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
5532                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
5533                                       NETIF_F_HW_VLAN_STAG_RX);
5534                 else
5535                         features |= NETIF_F_HW_VLAN_CTAG_RX |
5536                                     NETIF_F_HW_VLAN_STAG_RX;
5537         }
5538 #ifdef CONFIG_BNXT_SRIOV
5539         if (BNXT_VF(bp)) {
5540                 if (bp->vf.vlan) {
5541                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
5542                                       NETIF_F_HW_VLAN_STAG_RX);
5543                 }
5544         }
5545 #endif
5546         return features;
5547 }
5548
5549 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
5550 {
5551         struct bnxt *bp = netdev_priv(dev);
5552         u32 flags = bp->flags;
5553         u32 changes;
5554         int rc = 0;
5555         bool re_init = false;
5556         bool update_tpa = false;
5557
5558         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
5559         if ((features & NETIF_F_GRO) && (bp->pdev->revision > 0))
5560                 flags |= BNXT_FLAG_GRO;
5561         if (features & NETIF_F_LRO)
5562                 flags |= BNXT_FLAG_LRO;
5563
5564         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5565                 flags |= BNXT_FLAG_STRIP_VLAN;
5566
5567         if (features & NETIF_F_NTUPLE)
5568                 flags |= BNXT_FLAG_RFS;
5569
5570         changes = flags ^ bp->flags;
5571         if (changes & BNXT_FLAG_TPA) {
5572                 update_tpa = true;
5573                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
5574                     (flags & BNXT_FLAG_TPA) == 0)
5575                         re_init = true;
5576         }
5577
5578         if (changes & ~BNXT_FLAG_TPA)
5579                 re_init = true;
5580
5581         if (flags != bp->flags) {
5582                 u32 old_flags = bp->flags;
5583
5584                 bp->flags = flags;
5585
5586                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5587                         if (update_tpa)
5588                                 bnxt_set_ring_params(bp);
5589                         return rc;
5590                 }
5591
5592                 if (re_init) {
5593                         bnxt_close_nic(bp, false, false);
5594                         if (update_tpa)
5595                                 bnxt_set_ring_params(bp);
5596
5597                         return bnxt_open_nic(bp, false, false);
5598                 }
5599                 if (update_tpa) {
5600                         rc = bnxt_set_tpa(bp,
5601                                           (flags & BNXT_FLAG_TPA) ?
5602                                           true : false);
5603                         if (rc)
5604                                 bp->flags = old_flags;
5605                 }
5606         }
5607         return rc;
5608 }
5609
5610 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
5611 {
5612         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
5613         int i = bnapi->index;
5614
5615         if (!txr)
5616                 return;
5617
5618         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
5619                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
5620                     txr->tx_cons);
5621 }
5622
5623 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
5624 {
5625         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
5626         int i = bnapi->index;
5627
5628         if (!rxr)
5629                 return;
5630
5631         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
5632                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
5633                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
5634                     rxr->rx_sw_agg_prod);
5635 }
5636
5637 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
5638 {
5639         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5640         int i = bnapi->index;
5641
5642         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
5643                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
5644 }
5645
5646 static void bnxt_dbg_dump_states(struct bnxt *bp)
5647 {
5648         int i;
5649         struct bnxt_napi *bnapi;
5650
5651         for (i = 0; i < bp->cp_nr_rings; i++) {
5652                 bnapi = bp->bnapi[i];
5653                 if (netif_msg_drv(bp)) {
5654                         bnxt_dump_tx_sw_state(bnapi);
5655                         bnxt_dump_rx_sw_state(bnapi);
5656                         bnxt_dump_cp_sw_state(bnapi);
5657                 }
5658         }
5659 }
5660
5661 static void bnxt_reset_task(struct bnxt *bp, bool silent)
5662 {
5663         if (!silent)
5664                 bnxt_dbg_dump_states(bp);
5665         if (netif_running(bp->dev)) {
5666                 bnxt_close_nic(bp, false, false);
5667                 bnxt_open_nic(bp, false, false);
5668         }
5669 }
5670
5671 static void bnxt_tx_timeout(struct net_device *dev)
5672 {
5673         struct bnxt *bp = netdev_priv(dev);
5674
5675         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
5676         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
5677         schedule_work(&bp->sp_task);
5678 }
5679
5680 #ifdef CONFIG_NET_POLL_CONTROLLER
5681 static void bnxt_poll_controller(struct net_device *dev)
5682 {
5683         struct bnxt *bp = netdev_priv(dev);
5684         int i;
5685
5686         for (i = 0; i < bp->cp_nr_rings; i++) {
5687                 struct bnxt_irq *irq = &bp->irq_tbl[i];
5688
5689                 disable_irq(irq->vector);
5690                 irq->handler(irq->vector, bp->bnapi[i]);
5691                 enable_irq(irq->vector);
5692         }
5693 }
5694 #endif
5695
5696 static void bnxt_timer(unsigned long data)
5697 {
5698         struct bnxt *bp = (struct bnxt *)data;
5699         struct net_device *dev = bp->dev;
5700
5701         if (!netif_running(dev))
5702                 return;
5703
5704         if (atomic_read(&bp->intr_sem) != 0)
5705                 goto bnxt_restart_timer;
5706
5707         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
5708                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
5709                 schedule_work(&bp->sp_task);
5710         }
5711 bnxt_restart_timer:
5712         mod_timer(&bp->timer, jiffies + bp->current_interval);
5713 }
5714
5715 /* Only called from bnxt_sp_task() */
5716 static void bnxt_reset(struct bnxt *bp, bool silent)
5717 {
5718         /* bnxt_reset_task() calls bnxt_close_nic() which waits
5719          * for BNXT_STATE_IN_SP_TASK to clear.
5720          * If there is a parallel dev_close(), bnxt_close() may be holding
5721          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
5722          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
5723          */
5724         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5725         rtnl_lock();
5726         if (test_bit(BNXT_STATE_OPEN, &bp->state))
5727                 bnxt_reset_task(bp, silent);
5728         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5729         rtnl_unlock();
5730 }
5731
5732 static void bnxt_cfg_ntp_filters(struct bnxt *);
5733
5734 static void bnxt_sp_task(struct work_struct *work)
5735 {
5736         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
5737         int rc;
5738
5739         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5740         smp_mb__after_atomic();
5741         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5742                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5743                 return;
5744         }
5745
5746         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
5747                 bnxt_cfg_rx_mode(bp);
5748
5749         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
5750                 bnxt_cfg_ntp_filters(bp);
5751         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
5752                 rc = bnxt_update_link(bp, true);
5753                 if (rc)
5754                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
5755                                    rc);
5756         }
5757         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
5758                 bnxt_hwrm_exec_fwd_req(bp);
5759         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
5760                 bnxt_hwrm_tunnel_dst_port_alloc(
5761                         bp, bp->vxlan_port,
5762                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5763         }
5764         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
5765                 bnxt_hwrm_tunnel_dst_port_free(
5766                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5767         }
5768         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
5769                 bnxt_reset(bp, false);
5770
5771         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
5772                 bnxt_reset(bp, true);
5773
5774         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event))
5775                 bnxt_get_port_module_status(bp);
5776
5777         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
5778                 bnxt_hwrm_port_qstats(bp);
5779
5780         smp_mb__before_atomic();
5781         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5782 }
5783
5784 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
5785 {
5786         int rc;
5787         struct bnxt *bp = netdev_priv(dev);
5788
5789         SET_NETDEV_DEV(dev, &pdev->dev);
5790
5791         /* enable device (incl. PCI PM wakeup), and bus-mastering */
5792         rc = pci_enable_device(pdev);
5793         if (rc) {
5794                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
5795                 goto init_err;
5796         }
5797
5798         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
5799                 dev_err(&pdev->dev,
5800                         "Cannot find PCI device base address, aborting\n");
5801                 rc = -ENODEV;
5802                 goto init_err_disable;
5803         }
5804
5805         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
5806         if (rc) {
5807                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
5808                 goto init_err_disable;
5809         }
5810
5811         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
5812             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
5813                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
5814                 goto init_err_disable;
5815         }
5816
5817         pci_set_master(pdev);
5818
5819         bp->dev = dev;
5820         bp->pdev = pdev;
5821
5822         bp->bar0 = pci_ioremap_bar(pdev, 0);
5823         if (!bp->bar0) {
5824                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
5825                 rc = -ENOMEM;
5826                 goto init_err_release;
5827         }
5828
5829         bp->bar1 = pci_ioremap_bar(pdev, 2);
5830         if (!bp->bar1) {
5831                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
5832                 rc = -ENOMEM;
5833                 goto init_err_release;
5834         }
5835
5836         bp->bar2 = pci_ioremap_bar(pdev, 4);
5837         if (!bp->bar2) {
5838                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
5839                 rc = -ENOMEM;
5840                 goto init_err_release;
5841         }
5842
5843         pci_enable_pcie_error_reporting(pdev);
5844
5845         INIT_WORK(&bp->sp_task, bnxt_sp_task);
5846
5847         spin_lock_init(&bp->ntp_fltr_lock);
5848
5849         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
5850         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
5851
5852         /* tick values in micro seconds */
5853         bp->rx_coal_ticks = 12;
5854         bp->rx_coal_bufs = 30;
5855         bp->rx_coal_ticks_irq = 1;
5856         bp->rx_coal_bufs_irq = 2;
5857
5858         bp->tx_coal_ticks = 25;
5859         bp->tx_coal_bufs = 30;
5860         bp->tx_coal_ticks_irq = 2;
5861         bp->tx_coal_bufs_irq = 2;
5862
5863         init_timer(&bp->timer);
5864         bp->timer.data = (unsigned long)bp;
5865         bp->timer.function = bnxt_timer;
5866         bp->current_interval = BNXT_TIMER_INTERVAL;
5867
5868         clear_bit(BNXT_STATE_OPEN, &bp->state);
5869
5870         return 0;
5871
5872 init_err_release:
5873         if (bp->bar2) {
5874                 pci_iounmap(pdev, bp->bar2);
5875                 bp->bar2 = NULL;
5876         }
5877
5878         if (bp->bar1) {
5879                 pci_iounmap(pdev, bp->bar1);
5880                 bp->bar1 = NULL;
5881         }
5882
5883         if (bp->bar0) {
5884                 pci_iounmap(pdev, bp->bar0);
5885                 bp->bar0 = NULL;
5886         }
5887
5888         pci_release_regions(pdev);
5889
5890 init_err_disable:
5891         pci_disable_device(pdev);
5892
5893 init_err:
5894         return rc;
5895 }
5896
5897 /* rtnl_lock held */
5898 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
5899 {
5900         struct sockaddr *addr = p;
5901         struct bnxt *bp = netdev_priv(dev);
5902         int rc = 0;
5903
5904         if (!is_valid_ether_addr(addr->sa_data))
5905                 return -EADDRNOTAVAIL;
5906
5907         rc = bnxt_approve_mac(bp, addr->sa_data);
5908         if (rc)
5909                 return rc;
5910
5911         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
5912                 return 0;
5913
5914         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5915         if (netif_running(dev)) {
5916                 bnxt_close_nic(bp, false, false);
5917                 rc = bnxt_open_nic(bp, false, false);
5918         }
5919
5920         return rc;
5921 }
5922
5923 /* rtnl_lock held */
5924 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
5925 {
5926         struct bnxt *bp = netdev_priv(dev);
5927
5928         if (new_mtu < 60 || new_mtu > 9000)
5929                 return -EINVAL;
5930
5931         if (netif_running(dev))
5932                 bnxt_close_nic(bp, false, false);
5933
5934         dev->mtu = new_mtu;
5935         bnxt_set_ring_params(bp);
5936
5937         if (netif_running(dev))
5938                 return bnxt_open_nic(bp, false, false);
5939
5940         return 0;
5941 }
5942
5943 static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
5944                          struct tc_to_netdev *ntc)
5945 {
5946         struct bnxt *bp = netdev_priv(dev);
5947         u8 tc;
5948
5949         if (ntc->type != TC_SETUP_MQPRIO)
5950                 return -EINVAL;
5951
5952         tc = ntc->tc;
5953
5954         if (tc > bp->max_tc) {
5955                 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n",
5956                            tc, bp->max_tc);
5957                 return -EINVAL;
5958         }
5959
5960         if (netdev_get_num_tc(dev) == tc)
5961                 return 0;
5962
5963         if (tc) {
5964                 int max_rx_rings, max_tx_rings, rc;
5965                 bool sh = false;
5966
5967                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5968                         sh = true;
5969
5970                 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
5971                 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings)
5972                         return -ENOMEM;
5973         }
5974
5975         /* Needs to close the device and do hw resource re-allocations */
5976         if (netif_running(bp->dev))
5977                 bnxt_close_nic(bp, true, false);
5978
5979         if (tc) {
5980                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
5981                 netdev_set_num_tc(dev, tc);
5982         } else {
5983                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
5984                 netdev_reset_tc(dev);
5985         }
5986         bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
5987         bp->num_stat_ctxs = bp->cp_nr_rings;
5988
5989         if (netif_running(bp->dev))
5990                 return bnxt_open_nic(bp, true, false);
5991
5992         return 0;
5993 }
5994
5995 #ifdef CONFIG_RFS_ACCEL
5996 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
5997                             struct bnxt_ntuple_filter *f2)
5998 {
5999         struct flow_keys *keys1 = &f1->fkeys;
6000         struct flow_keys *keys2 = &f2->fkeys;
6001
6002         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
6003             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
6004             keys1->ports.ports == keys2->ports.ports &&
6005             keys1->basic.ip_proto == keys2->basic.ip_proto &&
6006             keys1->basic.n_proto == keys2->basic.n_proto &&
6007             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr))
6008                 return true;
6009
6010         return false;
6011 }
6012
6013 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
6014                               u16 rxq_index, u32 flow_id)
6015 {
6016         struct bnxt *bp = netdev_priv(dev);
6017         struct bnxt_ntuple_filter *fltr, *new_fltr;
6018         struct flow_keys *fkeys;
6019         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
6020         int rc = 0, idx, bit_id;
6021         struct hlist_head *head;
6022
6023         if (skb->encapsulation)
6024                 return -EPROTONOSUPPORT;
6025
6026         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
6027         if (!new_fltr)
6028                 return -ENOMEM;
6029
6030         fkeys = &new_fltr->fkeys;
6031         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
6032                 rc = -EPROTONOSUPPORT;
6033                 goto err_free;
6034         }
6035
6036         if ((fkeys->basic.n_proto != htons(ETH_P_IP)) ||
6037             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
6038              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
6039                 rc = -EPROTONOSUPPORT;
6040                 goto err_free;
6041         }
6042
6043         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
6044
6045         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
6046         head = &bp->ntp_fltr_hash_tbl[idx];
6047         rcu_read_lock();
6048         hlist_for_each_entry_rcu(fltr, head, hash) {
6049                 if (bnxt_fltr_match(fltr, new_fltr)) {
6050                         rcu_read_unlock();
6051                         rc = 0;
6052                         goto err_free;
6053                 }
6054         }
6055         rcu_read_unlock();
6056
6057         spin_lock_bh(&bp->ntp_fltr_lock);
6058         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
6059                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
6060         if (bit_id < 0) {
6061                 spin_unlock_bh(&bp->ntp_fltr_lock);
6062                 rc = -ENOMEM;
6063                 goto err_free;
6064         }
6065
6066         new_fltr->sw_id = (u16)bit_id;
6067         new_fltr->flow_id = flow_id;
6068         new_fltr->rxq = rxq_index;
6069         hlist_add_head_rcu(&new_fltr->hash, head);
6070         bp->ntp_fltr_count++;
6071         spin_unlock_bh(&bp->ntp_fltr_lock);
6072
6073         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
6074         schedule_work(&bp->sp_task);
6075
6076         return new_fltr->sw_id;
6077
6078 err_free:
6079         kfree(new_fltr);
6080         return rc;
6081 }
6082
6083 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
6084 {
6085         int i;
6086
6087         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
6088                 struct hlist_head *head;
6089                 struct hlist_node *tmp;
6090                 struct bnxt_ntuple_filter *fltr;
6091                 int rc;
6092
6093                 head = &bp->ntp_fltr_hash_tbl[i];
6094                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
6095                         bool del = false;
6096
6097                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
6098                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
6099                                                         fltr->flow_id,
6100                                                         fltr->sw_id)) {
6101                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
6102                                                                          fltr);
6103                                         del = true;
6104                                 }
6105                         } else {
6106                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
6107                                                                        fltr);
6108                                 if (rc)
6109                                         del = true;
6110                                 else
6111                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
6112                         }
6113
6114                         if (del) {
6115                                 spin_lock_bh(&bp->ntp_fltr_lock);
6116                                 hlist_del_rcu(&fltr->hash);
6117                                 bp->ntp_fltr_count--;
6118                                 spin_unlock_bh(&bp->ntp_fltr_lock);
6119                                 synchronize_rcu();
6120                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
6121                                 kfree(fltr);
6122                         }
6123                 }
6124         }
6125         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
6126                 netdev_info(bp->dev, "Receive PF driver unload event!");
6127 }
6128
6129 #else
6130
6131 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
6132 {
6133 }
6134
6135 #endif /* CONFIG_RFS_ACCEL */
6136
6137 static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6138                                 __be16 port)
6139 {
6140         struct bnxt *bp = netdev_priv(dev);
6141
6142         if (!netif_running(dev))
6143                 return;
6144
6145         if (sa_family != AF_INET6 && sa_family != AF_INET)
6146                 return;
6147
6148         if (bp->vxlan_port_cnt && bp->vxlan_port != port)
6149                 return;
6150
6151         bp->vxlan_port_cnt++;
6152         if (bp->vxlan_port_cnt == 1) {
6153                 bp->vxlan_port = port;
6154                 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
6155                 schedule_work(&bp->sp_task);
6156         }
6157 }
6158
6159 static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6160                                 __be16 port)
6161 {
6162         struct bnxt *bp = netdev_priv(dev);
6163
6164         if (!netif_running(dev))
6165                 return;
6166
6167         if (sa_family != AF_INET6 && sa_family != AF_INET)
6168                 return;
6169
6170         if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
6171                 bp->vxlan_port_cnt--;
6172
6173                 if (bp->vxlan_port_cnt == 0) {
6174                         set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
6175                         schedule_work(&bp->sp_task);
6176                 }
6177         }
6178 }
6179
6180 static const struct net_device_ops bnxt_netdev_ops = {
6181         .ndo_open               = bnxt_open,
6182         .ndo_start_xmit         = bnxt_start_xmit,
6183         .ndo_stop               = bnxt_close,
6184         .ndo_get_stats64        = bnxt_get_stats64,
6185         .ndo_set_rx_mode        = bnxt_set_rx_mode,
6186         .ndo_do_ioctl           = bnxt_ioctl,
6187         .ndo_validate_addr      = eth_validate_addr,
6188         .ndo_set_mac_address    = bnxt_change_mac_addr,
6189         .ndo_change_mtu         = bnxt_change_mtu,
6190         .ndo_fix_features       = bnxt_fix_features,
6191         .ndo_set_features       = bnxt_set_features,
6192         .ndo_tx_timeout         = bnxt_tx_timeout,
6193 #ifdef CONFIG_BNXT_SRIOV
6194         .ndo_get_vf_config      = bnxt_get_vf_config,
6195         .ndo_set_vf_mac         = bnxt_set_vf_mac,
6196         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
6197         .ndo_set_vf_rate        = bnxt_set_vf_bw,
6198         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
6199         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
6200 #endif
6201 #ifdef CONFIG_NET_POLL_CONTROLLER
6202         .ndo_poll_controller    = bnxt_poll_controller,
6203 #endif
6204         .ndo_setup_tc           = bnxt_setup_tc,
6205 #ifdef CONFIG_RFS_ACCEL
6206         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
6207 #endif
6208         .ndo_add_vxlan_port     = bnxt_add_vxlan_port,
6209         .ndo_del_vxlan_port     = bnxt_del_vxlan_port,
6210 #ifdef CONFIG_NET_RX_BUSY_POLL
6211         .ndo_busy_poll          = bnxt_busy_poll,
6212 #endif
6213 };
6214
6215 static void bnxt_remove_one(struct pci_dev *pdev)
6216 {
6217         struct net_device *dev = pci_get_drvdata(pdev);
6218         struct bnxt *bp = netdev_priv(dev);
6219
6220         if (BNXT_PF(bp))
6221                 bnxt_sriov_disable(bp);
6222
6223         pci_disable_pcie_error_reporting(pdev);
6224         unregister_netdev(dev);
6225         cancel_work_sync(&bp->sp_task);
6226         bp->sp_event = 0;
6227
6228         bnxt_hwrm_func_drv_unrgtr(bp);
6229         bnxt_free_hwrm_resources(bp);
6230         pci_iounmap(pdev, bp->bar2);
6231         pci_iounmap(pdev, bp->bar1);
6232         pci_iounmap(pdev, bp->bar0);
6233         free_netdev(dev);
6234
6235         pci_release_regions(pdev);
6236         pci_disable_device(pdev);
6237 }
6238
6239 static int bnxt_probe_phy(struct bnxt *bp)
6240 {
6241         int rc = 0;
6242         struct bnxt_link_info *link_info = &bp->link_info;
6243
6244         rc = bnxt_hwrm_phy_qcaps(bp);
6245         if (rc) {
6246                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
6247                            rc);
6248                 return rc;
6249         }
6250
6251         rc = bnxt_update_link(bp, false);
6252         if (rc) {
6253                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
6254                            rc);
6255                 return rc;
6256         }
6257
6258         /*initialize the ethool setting copy with NVM settings */
6259         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
6260                 link_info->autoneg = BNXT_AUTONEG_SPEED;
6261                 if (bp->hwrm_spec_code >= 0x10201) {
6262                         if (link_info->auto_pause_setting &
6263                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
6264                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6265                 } else {
6266                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6267                 }
6268                 link_info->advertising = link_info->auto_link_speeds;
6269         } else {
6270                 link_info->req_link_speed = link_info->force_link_speed;
6271                 link_info->req_duplex = link_info->duplex_setting;
6272         }
6273         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
6274                 link_info->req_flow_ctrl =
6275                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
6276         else
6277                 link_info->req_flow_ctrl = link_info->force_pause_setting;
6278         return rc;
6279 }
6280
6281 static int bnxt_get_max_irq(struct pci_dev *pdev)
6282 {
6283         u16 ctrl;
6284
6285         if (!pdev->msix_cap)
6286                 return 1;
6287
6288         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
6289         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
6290 }
6291
6292 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
6293                                 int *max_cp)
6294 {
6295         int max_ring_grps = 0;
6296
6297 #ifdef CONFIG_BNXT_SRIOV
6298         if (!BNXT_PF(bp)) {
6299                 *max_tx = bp->vf.max_tx_rings;
6300                 *max_rx = bp->vf.max_rx_rings;
6301                 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
6302                 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
6303                 max_ring_grps = bp->vf.max_hw_ring_grps;
6304         } else
6305 #endif
6306         {
6307                 *max_tx = bp->pf.max_tx_rings;
6308                 *max_rx = bp->pf.max_rx_rings;
6309                 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
6310                 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
6311                 max_ring_grps = bp->pf.max_hw_ring_grps;
6312         }
6313
6314         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6315                 *max_rx >>= 1;
6316         *max_rx = min_t(int, *max_rx, max_ring_grps);
6317 }
6318
6319 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
6320 {
6321         int rx, tx, cp;
6322
6323         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
6324         if (!rx || !tx || !cp)
6325                 return -ENOMEM;
6326
6327         *max_rx = rx;
6328         *max_tx = tx;
6329         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
6330 }
6331
6332 static int bnxt_set_dflt_rings(struct bnxt *bp)
6333 {
6334         int dflt_rings, max_rx_rings, max_tx_rings, rc;
6335         bool sh = true;
6336
6337         if (sh)
6338                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
6339         dflt_rings = netif_get_num_default_rss_queues();
6340         rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6341         if (rc)
6342                 return rc;
6343         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
6344         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
6345         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
6346         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
6347                                bp->tx_nr_rings + bp->rx_nr_rings;
6348         bp->num_stat_ctxs = bp->cp_nr_rings;
6349         return rc;
6350 }
6351
6352 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
6353 {
6354         enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
6355         enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
6356
6357         if (pcie_get_minimum_link(bp->pdev, &speed, &width) ||
6358             speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
6359                 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
6360         else
6361                 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
6362                             speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
6363                             speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
6364                             speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
6365                             "Unknown", width);
6366 }
6367
6368 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6369 {
6370         static int version_printed;
6371         struct net_device *dev;
6372         struct bnxt *bp;
6373         int rc, max_irqs;
6374
6375         if (version_printed++ == 0)
6376                 pr_info("%s", version);
6377
6378         max_irqs = bnxt_get_max_irq(pdev);
6379         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
6380         if (!dev)
6381                 return -ENOMEM;
6382
6383         bp = netdev_priv(dev);
6384
6385         if (bnxt_vf_pciid(ent->driver_data))
6386                 bp->flags |= BNXT_FLAG_VF;
6387
6388         if (pdev->msix_cap)
6389                 bp->flags |= BNXT_FLAG_MSIX_CAP;
6390
6391         rc = bnxt_init_board(pdev, dev);
6392         if (rc < 0)
6393                 goto init_err_free;
6394
6395         dev->netdev_ops = &bnxt_netdev_ops;
6396         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
6397         dev->ethtool_ops = &bnxt_ethtool_ops;
6398
6399         pci_set_drvdata(pdev, dev);
6400
6401         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6402                            NETIF_F_TSO | NETIF_F_TSO6 |
6403                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6404                            NETIF_F_GSO_IPXIP4 |
6405                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6406                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
6407                            NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO;
6408
6409         dev->hw_enc_features =
6410                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6411                         NETIF_F_TSO | NETIF_F_TSO6 |
6412                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6413                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6414                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
6415         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
6416                                     NETIF_F_GSO_GRE_CSUM;
6417         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
6418         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
6419                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
6420         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
6421         dev->priv_flags |= IFF_UNICAST_FLT;
6422
6423 #ifdef CONFIG_BNXT_SRIOV
6424         init_waitqueue_head(&bp->sriov_cfg_wait);
6425 #endif
6426         rc = bnxt_alloc_hwrm_resources(bp);
6427         if (rc)
6428                 goto init_err;
6429
6430         mutex_init(&bp->hwrm_cmd_lock);
6431         rc = bnxt_hwrm_ver_get(bp);
6432         if (rc)
6433                 goto init_err;
6434
6435         bp->gro_func = bnxt_gro_func_5730x;
6436
6437         rc = bnxt_hwrm_func_drv_rgtr(bp);
6438         if (rc)
6439                 goto init_err;
6440
6441         /* Get the MAX capabilities for this function */
6442         rc = bnxt_hwrm_func_qcaps(bp);
6443         if (rc) {
6444                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
6445                            rc);
6446                 rc = -1;
6447                 goto init_err;
6448         }
6449
6450         rc = bnxt_hwrm_queue_qportcfg(bp);
6451         if (rc) {
6452                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
6453                            rc);
6454                 rc = -1;
6455                 goto init_err;
6456         }
6457
6458         bnxt_hwrm_func_qcfg(bp);
6459
6460         bnxt_set_tpa_flags(bp);
6461         bnxt_set_ring_params(bp);
6462         if (BNXT_PF(bp))
6463                 bp->pf.max_irqs = max_irqs;
6464 #if defined(CONFIG_BNXT_SRIOV)
6465         else
6466                 bp->vf.max_irqs = max_irqs;
6467 #endif
6468         bnxt_set_dflt_rings(bp);
6469
6470         if (BNXT_PF(bp)) {
6471                 dev->hw_features |= NETIF_F_NTUPLE;
6472                 if (bnxt_rfs_capable(bp)) {
6473                         bp->flags |= BNXT_FLAG_RFS;
6474                         dev->features |= NETIF_F_NTUPLE;
6475                 }
6476         }
6477
6478         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
6479                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
6480
6481         rc = bnxt_probe_phy(bp);
6482         if (rc)
6483                 goto init_err;
6484
6485         rc = register_netdev(dev);
6486         if (rc)
6487                 goto init_err;
6488
6489         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
6490                     board_info[ent->driver_data].name,
6491                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
6492
6493         bnxt_parse_log_pcie_link(bp);
6494
6495         return 0;
6496
6497 init_err:
6498         pci_iounmap(pdev, bp->bar0);
6499         pci_release_regions(pdev);
6500         pci_disable_device(pdev);
6501
6502 init_err_free:
6503         free_netdev(dev);
6504         return rc;
6505 }
6506
6507 /**
6508  * bnxt_io_error_detected - called when PCI error is detected
6509  * @pdev: Pointer to PCI device
6510  * @state: The current pci connection state
6511  *
6512  * This function is called after a PCI bus error affecting
6513  * this device has been detected.
6514  */
6515 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
6516                                                pci_channel_state_t state)
6517 {
6518         struct net_device *netdev = pci_get_drvdata(pdev);
6519
6520         netdev_info(netdev, "PCI I/O error detected\n");
6521
6522         rtnl_lock();
6523         netif_device_detach(netdev);
6524
6525         if (state == pci_channel_io_perm_failure) {
6526                 rtnl_unlock();
6527                 return PCI_ERS_RESULT_DISCONNECT;
6528         }
6529
6530         if (netif_running(netdev))
6531                 bnxt_close(netdev);
6532
6533         pci_disable_device(pdev);
6534         rtnl_unlock();
6535
6536         /* Request a slot slot reset. */
6537         return PCI_ERS_RESULT_NEED_RESET;
6538 }
6539
6540 /**
6541  * bnxt_io_slot_reset - called after the pci bus has been reset.
6542  * @pdev: Pointer to PCI device
6543  *
6544  * Restart the card from scratch, as if from a cold-boot.
6545  * At this point, the card has exprienced a hard reset,
6546  * followed by fixups by BIOS, and has its config space
6547  * set up identically to what it was at cold boot.
6548  */
6549 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
6550 {
6551         struct net_device *netdev = pci_get_drvdata(pdev);
6552         struct bnxt *bp = netdev_priv(netdev);
6553         int err = 0;
6554         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
6555
6556         netdev_info(bp->dev, "PCI Slot Reset\n");
6557
6558         rtnl_lock();
6559
6560         if (pci_enable_device(pdev)) {
6561                 dev_err(&pdev->dev,
6562                         "Cannot re-enable PCI device after reset.\n");
6563         } else {
6564                 pci_set_master(pdev);
6565
6566                 if (netif_running(netdev))
6567                         err = bnxt_open(netdev);
6568
6569                 if (!err)
6570                         result = PCI_ERS_RESULT_RECOVERED;
6571         }
6572
6573         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
6574                 dev_close(netdev);
6575
6576         rtnl_unlock();
6577
6578         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6579         if (err) {
6580                 dev_err(&pdev->dev,
6581                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
6582                          err); /* non-fatal, continue */
6583         }
6584
6585         return PCI_ERS_RESULT_RECOVERED;
6586 }
6587
6588 /**
6589  * bnxt_io_resume - called when traffic can start flowing again.
6590  * @pdev: Pointer to PCI device
6591  *
6592  * This callback is called when the error recovery driver tells
6593  * us that its OK to resume normal operation.
6594  */
6595 static void bnxt_io_resume(struct pci_dev *pdev)
6596 {
6597         struct net_device *netdev = pci_get_drvdata(pdev);
6598
6599         rtnl_lock();
6600
6601         netif_device_attach(netdev);
6602
6603         rtnl_unlock();
6604 }
6605
6606 static const struct pci_error_handlers bnxt_err_handler = {
6607         .error_detected = bnxt_io_error_detected,
6608         .slot_reset     = bnxt_io_slot_reset,
6609         .resume         = bnxt_io_resume
6610 };
6611
6612 static struct pci_driver bnxt_pci_driver = {
6613         .name           = DRV_MODULE_NAME,
6614         .id_table       = bnxt_pci_tbl,
6615         .probe          = bnxt_init_one,
6616         .remove         = bnxt_remove_one,
6617         .err_handler    = &bnxt_err_handler,
6618 #if defined(CONFIG_BNXT_SRIOV)
6619         .sriov_configure = bnxt_sriov_configure,
6620 #endif
6621 };
6622
6623 module_pci_driver(bnxt_pci_driver);