drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2014 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.12.1-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69         [board_X550_vf]  = &ixgbevf_X550_vf_info,
  70         [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info,
  71 };
  72
  73 /* ixgbevf_pci_tbl - PCI Device ID Table
  74  *
  75  * Wildcard entries (PCI_ANY_ID) should come last
  76  * Last entry must be all 0s
  77  *
  78  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  79  *   Class, Class Mask, private data (not used) }
  80  */
  81 static const struct pci_device_id ixgbevf_pci_tbl[] = {
  82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
  83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
  84         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
  85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
  86         /* required last entry */
  87         {0, }
  88 };
  89 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  90
  91 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  92 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
  93 MODULE_LICENSE("GPL");
  94 MODULE_VERSION(DRV_VERSION);
  95
  96 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  97 static int debug = -1;
  98 module_param(debug, int, 0);
  99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 100
 101 /* forward decls */
 102 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
 103 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
 104 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
 105
 106 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
 107 {
 108         struct ixgbevf_adapter *adapter = hw->back;
 109
 110         if (!hw->hw_addr)
 111                 return;
 112         hw->hw_addr = NULL;
 113         dev_err(&adapter->pdev->dev, "Adapter removed\n");
 114         if (test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
 115                 schedule_work(&adapter->watchdog_task);
 116 }
 117
 118 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
 119 {
 120         u32 value;
 121
 122         /* The following check not only optimizes a bit by not
 123          * performing a read on the status register when the
 124          * register just read was a status register read that
 125          * returned IXGBE_FAILED_READ_REG. It also blocks any
 126          * potential recursion.
 127          */
 128         if (reg == IXGBE_VFSTATUS) {
 129                 ixgbevf_remove_adapter(hw);
 130                 return;
 131         }
 132         value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
 133         if (value == IXGBE_FAILED_READ_REG)
 134                 ixgbevf_remove_adapter(hw);
 135 }
 136
 137 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
 138 {
 139         u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
 140         u32 value;
 141
 142         if (IXGBE_REMOVED(reg_addr))
 143                 return IXGBE_FAILED_READ_REG;
 144         value = readl(reg_addr + reg);
 145         if (unlikely(value == IXGBE_FAILED_READ_REG))
 146                 ixgbevf_check_remove(hw, reg);
 147         return value;
 148 }
 149
 150 /**
 151  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 152  * @adapter: pointer to adapter struct
 153  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 154  * @queue: queue to map the corresponding interrupt to
 155  * @msix_vector: the vector to map to the corresponding queue
 156  */
 157 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 158                              u8 queue, u8 msix_vector)
 159 {
 160         u32 ivar, index;
 161         struct ixgbe_hw *hw = &adapter->hw;
 162         if (direction == -1) {
 163                 /* other causes */
 164                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 165                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 166                 ivar &= ~0xFF;
 167                 ivar |= msix_vector;
 168                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 169         } else {
 170                 /* tx or rx causes */
 171                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 172                 index = ((16 * (queue & 1)) + (8 * direction));
 173                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 174                 ivar &= ~(0xFF << index);
 175                 ivar |= (msix_vector << index);
 176                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 177         }
 178 }
 179
 180 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 181                                         struct ixgbevf_tx_buffer *tx_buffer)
 182 {
 183         if (tx_buffer->skb) {
 184                 dev_kfree_skb_any(tx_buffer->skb);
 185                 if (dma_unmap_len(tx_buffer, len))
 186                         dma_unmap_single(tx_ring->dev,
 187                                          dma_unmap_addr(tx_buffer, dma),
 188                                          dma_unmap_len(tx_buffer, len),
 189                                          DMA_TO_DEVICE);
 190         } else if (dma_unmap_len(tx_buffer, len)) {
 191                 dma_unmap_page(tx_ring->dev,
 192                                dma_unmap_addr(tx_buffer, dma),
 193                                dma_unmap_len(tx_buffer, len),
 194                                DMA_TO_DEVICE);
 195         }
 196         tx_buffer->next_to_watch = NULL;
 197         tx_buffer->skb = NULL;
 198         dma_unmap_len_set(tx_buffer, len, 0);
 199         /* tx_buffer must be completely set up in the transmit path */
 200 }
 201
 202 #define IXGBE_MAX_TXD_PWR       14
 203 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 204
 205 /* Tx Descriptors needed, worst case */
 206 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 207 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 208
 209 static void ixgbevf_tx_timeout(struct net_device *netdev);
 210
 211 /**
 212  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 213  * @q_vector: board private structure
 214  * @tx_ring: tx ring to clean
 215  **/
 216 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 217                                  struct ixgbevf_ring *tx_ring)
 218 {
 219         struct ixgbevf_adapter *adapter = q_vector->adapter;
 220         struct ixgbevf_tx_buffer *tx_buffer;
 221         union ixgbe_adv_tx_desc *tx_desc;
 222         unsigned int total_bytes = 0, total_packets = 0;
 223         unsigned int budget = tx_ring->count / 2;
 224         unsigned int i = tx_ring->next_to_clean;
 225
 226         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 227                 return true;
 228
 229         tx_buffer = &tx_ring->tx_buffer_info[i];
 230         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 231         i -= tx_ring->count;
 232
 233         do {
 234                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
 235
 236                 /* if next_to_watch is not set then there is no work pending */
 237                 if (!eop_desc)
 238                         break;
 239
 240                 /* prevent any other reads prior to eop_desc */
 241                 read_barrier_depends();
 242
 243                 /* if DD is not set pending work has not been completed */
 244                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 245                         break;
 246
 247                 /* clear next_to_watch to prevent false hangs */
 248                 tx_buffer->next_to_watch = NULL;
 249
 250                 /* update the statistics for this packet */
 251                 total_bytes += tx_buffer->bytecount;
 252                 total_packets += tx_buffer->gso_segs;
 253
 254                 /* free the skb */
 255                 dev_kfree_skb_any(tx_buffer->skb);
 256
 257                 /* unmap skb header data */
 258                 dma_unmap_single(tx_ring->dev,
 259                                  dma_unmap_addr(tx_buffer, dma),
 260                                  dma_unmap_len(tx_buffer, len),
 261                                  DMA_TO_DEVICE);
 262
 263                 /* clear tx_buffer data */
 264                 tx_buffer->skb = NULL;
 265                 dma_unmap_len_set(tx_buffer, len, 0);
 266
 267                 /* unmap remaining buffers */
 268                 while (tx_desc != eop_desc) {
 269                         tx_buffer++;
 270                         tx_desc++;
 271                         i++;
 272                         if (unlikely(!i)) {
 273                                 i -= tx_ring->count;
 274                                 tx_buffer = tx_ring->tx_buffer_info;
 275                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 276                         }
 277
 278                         /* unmap any remaining paged data */
 279                         if (dma_unmap_len(tx_buffer, len)) {
 280                                 dma_unmap_page(tx_ring->dev,
 281                                                dma_unmap_addr(tx_buffer, dma),
 282                                                dma_unmap_len(tx_buffer, len),
 283                                                DMA_TO_DEVICE);
 284                                 dma_unmap_len_set(tx_buffer, len, 0);
 285                         }
 286                 }
 287
 288                 /* move us one more past the eop_desc for start of next pkt */
 289                 tx_buffer++;
 290                 tx_desc++;
 291                 i++;
 292                 if (unlikely(!i)) {
 293                         i -= tx_ring->count;
 294                         tx_buffer = tx_ring->tx_buffer_info;
 295                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 296                 }
 297
 298                 /* issue prefetch for next Tx descriptor */
 299                 prefetch(tx_desc);
 300
 301                 /* update budget accounting */
 302                 budget--;
 303         } while (likely(budget));
 304
 305         i += tx_ring->count;
 306         tx_ring->next_to_clean = i;
 307         u64_stats_update_begin(&tx_ring->syncp);
 308         tx_ring->stats.bytes += total_bytes;
 309         tx_ring->stats.packets += total_packets;
 310         u64_stats_update_end(&tx_ring->syncp);
 311         q_vector->tx.total_bytes += total_bytes;
 312         q_vector->tx.total_packets += total_packets;
 313
 314 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 315         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 316                      (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 317                 /* Make sure that anybody stopping the queue after this
 318                  * sees the new next_to_clean.
 319                  */
 320                 smp_mb();
 321
 322                 if (__netif_subqueue_stopped(tx_ring->netdev,
 323                                              tx_ring->queue_index) &&
 324                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 325                         netif_wake_subqueue(tx_ring->netdev,
 326                                             tx_ring->queue_index);
 327                         ++tx_ring->tx_stats.restart_queue;
 328                 }
 329         }
 330
 331         return !!budget;
 332 }
 333
 334 /**
 335  * ixgbevf_rx_skb - Helper function to determine proper Rx method
 336  * @q_vector: structure containing interrupt and ring information
 337  * @skb: packet to send up
 338  **/
 339 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
 340                            struct sk_buff *skb)
 341 {
 342 #ifdef CONFIG_NET_RX_BUSY_POLL
 343         skb_mark_napi_id(skb, &q_vector->napi);
 344
 345         if (ixgbevf_qv_busy_polling(q_vector)) {
 346                 netif_receive_skb(skb);
 347                 /* exit early if we busy polled */
 348                 return;
 349         }
 350 #endif /* CONFIG_NET_RX_BUSY_POLL */
 351
 352         napi_gro_receive(&q_vector->napi, skb);
 353 }
 354
 355 /* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 356  * @ring: structure containig ring specific data
 357  * @rx_desc: current Rx descriptor being processed
 358  * @skb: skb currently being received and modified
 359  */
 360 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
 361                                        union ixgbe_adv_rx_desc *rx_desc,
 362                                        struct sk_buff *skb)
 363 {
 364         skb_checksum_none_assert(skb);
 365
 366         /* Rx csum disabled */
 367         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 368                 return;
 369
 370         /* if IP and error */
 371         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
 372             ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
 373                 ring->rx_stats.csum_err++;
 374                 return;
 375         }
 376
 377         if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
 378                 return;
 379
 380         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
 381                 ring->rx_stats.csum_err++;
 382                 return;
 383         }
 384
 385         /* It must be a TCP or UDP packet with a valid checksum */
 386         skb->ip_summed = CHECKSUM_UNNECESSARY;
 387 }
 388
 389 /* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
 390  * @rx_ring: rx descriptor ring packet is being transacted on
 391  * @rx_desc: pointer to the EOP Rx descriptor
 392  * @skb: pointer to current skb being populated
 393  *
 394  * This function checks the ring, descriptor, and packet information in
 395  * order to populate the checksum, VLAN, protocol, and other fields within
 396  * the skb.
 397  */
 398 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
 399                                        union ixgbe_adv_rx_desc *rx_desc,
 400                                        struct sk_buff *skb)
 401 {
 402         ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
 403
 404         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
 405                 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
 406                 unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
 407
 408                 if (test_bit(vid & VLAN_VID_MASK, active_vlans))
 409                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
 410         }
 411
 412         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 413 }
 414
 415 /**
 416  * ixgbevf_is_non_eop - process handling of non-EOP buffers
 417  * @rx_ring: Rx ring being processed
 418  * @rx_desc: Rx descriptor for current buffer
 419  * @skb: current socket buffer containing buffer in progress
 420  *
 421  * This function updates next to clean.  If the buffer is an EOP buffer
 422  * this function exits returning false, otherwise it will place the
 423  * sk_buff in the next buffer to be chained and return true indicating
 424  * that this is in fact a non-EOP buffer.
 425  **/
 426 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
 427                                union ixgbe_adv_rx_desc *rx_desc)
 428 {
 429         u32 ntc = rx_ring->next_to_clean + 1;
 430
 431         /* fetch, update, and store next to clean */
 432         ntc = (ntc < rx_ring->count) ? ntc : 0;
 433         rx_ring->next_to_clean = ntc;
 434
 435         prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
 436
 437         if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
 438                 return false;
 439
 440         return true;
 441 }
 442
 443 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
 444                                       struct ixgbevf_rx_buffer *bi)
 445 {
 446         struct page *page = bi->page;
 447         dma_addr_t dma = bi->dma;
 448
 449         /* since we are recycling buffers we should seldom need to alloc */
 450         if (likely(page))
 451                 return true;
 452
 453         /* alloc new page for storage */
 454         page = dev_alloc_page();
 455         if (unlikely(!page)) {
 456                 rx_ring->rx_stats.alloc_rx_page_failed++;
 457                 return false;
 458         }
 459
 460         /* map page for use */
 461         dma = dma_map_page(rx_ring->dev, page, 0,
 462                            PAGE_SIZE, DMA_FROM_DEVICE);
 463
 464         /* if mapping failed free memory back to system since
 465          * there isn't much point in holding memory we can't use
 466          */
 467         if (dma_mapping_error(rx_ring->dev, dma)) {
 468                 __free_page(page);
 469
 470                 rx_ring->rx_stats.alloc_rx_buff_failed++;
 471                 return false;
 472         }
 473
 474         bi->dma = dma;
 475         bi->page = page;
 476         bi->page_offset = 0;
 477
 478         return true;
 479 }
 480
 481 /**
 482  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 483  * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
 484  * @cleaned_count: number of buffers to replace
 485  **/
 486 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
 487                                      u16 cleaned_count)
 488 {
 489         union ixgbe_adv_rx_desc *rx_desc;
 490         struct ixgbevf_rx_buffer *bi;
 491         unsigned int i = rx_ring->next_to_use;
 492
 493         /* nothing to do or no valid netdev defined */
 494         if (!cleaned_count || !rx_ring->netdev)
 495                 return;
 496
 497         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 498         bi = &rx_ring->rx_buffer_info[i];
 499         i -= rx_ring->count;
 500
 501         do {
 502                 if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
 503                         break;
 504
 505                 /* Refresh the desc even if pkt_addr didn't change
 506                  * because each write-back erases this info.
 507                  */
 508                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
 509
 510                 rx_desc++;
 511                 bi++;
 512                 i++;
 513                 if (unlikely(!i)) {
 514                         rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
 515                         bi = rx_ring->rx_buffer_info;
 516                         i -= rx_ring->count;
 517                 }
 518
 519                 /* clear the hdr_addr for the next_to_use descriptor */
 520                 rx_desc->read.hdr_addr = 0;
 521
 522                 cleaned_count--;
 523         } while (cleaned_count);
 524
 525         i += rx_ring->count;
 526
 527         if (rx_ring->next_to_use != i) {
 528                 /* record the next descriptor to use */
 529                 rx_ring->next_to_use = i;
 530
 531                 /* update next to alloc since we have filled the ring */
 532                 rx_ring->next_to_alloc = i;
 533
 534                 /* Force memory writes to complete before letting h/w
 535                  * know there are new descriptors to fetch.  (Only
 536                  * applicable for weak-ordered memory model archs,
 537                  * such as IA-64).
 538                  */
 539                 wmb();
 540                 ixgbevf_write_tail(rx_ring, i);
 541         }
 542 }
 543
 544 /* ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail
 545  * @rx_ring: rx descriptor ring packet is being transacted on
 546  * @skb: pointer to current skb being adjusted
 547  *
 548  * This function is an ixgbevf specific version of __pskb_pull_tail.  The
 549  * main difference between this version and the original function is that
 550  * this function can make several assumptions about the state of things
 551  * that allow for significant optimizations versus the standard function.
 552  * As a result we can do things like drop a frag and maintain an accurate
 553  * truesize for the skb.
 554  */
 555 static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring,
 556                               struct sk_buff *skb)
 557 {
 558         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
 559         unsigned char *va;
 560         unsigned int pull_len;
 561
 562         /* it is valid to use page_address instead of kmap since we are
 563          * working with pages allocated out of the lomem pool per
 564          * alloc_page(GFP_ATOMIC)
 565          */
 566         va = skb_frag_address(frag);
 567
 568         /* we need the header to contain the greater of either ETH_HLEN or
 569          * 60 bytes if the skb->len is less than 60 for skb_pad.
 570          */
 571         pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE);
 572
 573         /* align pull length to size of long to optimize memcpy performance */
 574         skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
 575
 576         /* update all of the pointers */
 577         skb_frag_size_sub(frag, pull_len);
 578         frag->page_offset += pull_len;
 579         skb->data_len -= pull_len;
 580         skb->tail += pull_len;
 581 }
 582
 583 /* ixgbevf_cleanup_headers - Correct corrupted or empty headers
 584  * @rx_ring: rx descriptor ring packet is being transacted on
 585  * @rx_desc: pointer to the EOP Rx descriptor
 586  * @skb: pointer to current skb being fixed
 587  *
 588  * Check for corrupted packet headers caused by senders on the local L2
 589  * embedded NIC switch not setting up their Tx Descriptors right.  These
 590  * should be very rare.
 591  *
 592  * Also address the case where we are pulling data in on pages only
 593  * and as such no data is present in the skb header.
 594  *
 595  * In addition if skb is not at least 60 bytes we need to pad it so that
 596  * it is large enough to qualify as a valid Ethernet frame.
 597  *
 598  * Returns true if an error was encountered and skb was freed.
 599  */
 600 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
 601                                     union ixgbe_adv_rx_desc *rx_desc,
 602                                     struct sk_buff *skb)
 603 {
 604         /* verify that the packet does not have any known errors */
 605         if (unlikely(ixgbevf_test_staterr(rx_desc,
 606                                           IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
 607                 struct net_device *netdev = rx_ring->netdev;
 608
 609                 if (!(netdev->features & NETIF_F_RXALL)) {
 610                         dev_kfree_skb_any(skb);
 611                         return true;
 612                 }
 613         }
 614
 615         /* place header in linear portion of buffer */
 616         if (skb_is_nonlinear(skb))
 617                 ixgbevf_pull_tail(rx_ring, skb);
 618
 619         /* if eth_skb_pad returns an error the skb was freed */
 620         if (eth_skb_pad(skb))
 621                 return true;
 622
 623         return false;
 624 }
 625
 626 /* ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
 627  * @rx_ring: rx descriptor ring to store buffers on
 628  * @old_buff: donor buffer to have page reused
 629  *
 630  * Synchronizes page for reuse by the adapter
 631  */
 632 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
 633                                   struct ixgbevf_rx_buffer *old_buff)
 634 {
 635         struct ixgbevf_rx_buffer *new_buff;
 636         u16 nta = rx_ring->next_to_alloc;
 637
 638         new_buff = &rx_ring->rx_buffer_info[nta];
 639
 640         /* update, and store next to alloc */
 641         nta++;
 642         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
 643
 644         /* transfer page from old buffer to new buffer */
 645         new_buff->page = old_buff->page;
 646         new_buff->dma = old_buff->dma;
 647         new_buff->page_offset = old_buff->page_offset;
 648
 649         /* sync the buffer for use by the device */
 650         dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
 651                                          new_buff->page_offset,
 652                                          IXGBEVF_RX_BUFSZ,
 653                                          DMA_FROM_DEVICE);
 654 }
 655
 656 static inline bool ixgbevf_page_is_reserved(struct page *page)
 657 {
 658         return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc;
 659 }
 660
 661 /* ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
 662  * @rx_ring: rx descriptor ring to transact packets on
 663  * @rx_buffer: buffer containing page to add
 664  * @rx_desc: descriptor containing length of buffer written by hardware
 665  * @skb: sk_buff to place the data into
 666  *
 667  * This function will add the data contained in rx_buffer->page to the skb.
 668  * This is done either through a direct copy if the data in the buffer is
 669  * less than the skb header size, otherwise it will just attach the page as
 670  * a frag to the skb.
 671  *
 672  * The function will then update the page offset if necessary and return
 673  * true if the buffer can be reused by the adapter.
 674  */
 675 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
 676                                 struct ixgbevf_rx_buffer *rx_buffer,
 677                                 union ixgbe_adv_rx_desc *rx_desc,
 678                                 struct sk_buff *skb)
 679 {
 680         struct page *page = rx_buffer->page;
 681         unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
 682 #if (PAGE_SIZE < 8192)
 683         unsigned int truesize = IXGBEVF_RX_BUFSZ;
 684 #else
 685         unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
 686 #endif
 687
 688         if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
 689                 unsigned char *va = page_address(page) + rx_buffer->page_offset;
 690
 691                 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
 692
 693                 /* page is not reserved, we can reuse buffer as is */
 694                 if (likely(!ixgbevf_page_is_reserved(page)))
 695                         return true;
 696
 697                 /* this page cannot be reused so discard it */
 698                 put_page(page);
 699                 return false;
 700         }
 701
 702         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
 703                         rx_buffer->page_offset, size, truesize);
 704
 705         /* avoid re-using remote pages */
 706         if (unlikely(ixgbevf_page_is_reserved(page)))
 707                 return false;
 708
 709 #if (PAGE_SIZE < 8192)
 710         /* if we are only owner of page we can reuse it */
 711         if (unlikely(page_count(page) != 1))
 712                 return false;
 713
 714         /* flip page offset to other buffer */
 715         rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
 716
 717 #else
 718         /* move offset up to the next cache line */
 719         rx_buffer->page_offset += truesize;
 720
 721         if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
 722                 return false;
 723
 724 #endif
 725         /* Even if we own the page, we are not allowed to use atomic_set()
 726          * This would break get_page_unless_zero() users.
 727          */
 728         atomic_inc(&page->_count);
 729
 730         return true;
 731 }
 732
 733 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
 734                                                union ixgbe_adv_rx_desc *rx_desc,
 735                                                struct sk_buff *skb)
 736 {
 737         struct ixgbevf_rx_buffer *rx_buffer;
 738         struct page *page;
 739
 740         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
 741         page = rx_buffer->page;
 742         prefetchw(page);
 743
 744         if (likely(!skb)) {
 745                 void *page_addr = page_address(page) +
 746                                   rx_buffer->page_offset;
 747
 748                 /* prefetch first cache line of first page */
 749                 prefetch(page_addr);
 750 #if L1_CACHE_BYTES < 128
 751                 prefetch(page_addr + L1_CACHE_BYTES);
 752 #endif
 753
 754                 /* allocate a skb to store the frags */
 755                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 756                                                 IXGBEVF_RX_HDR_SIZE);
 757                 if (unlikely(!skb)) {
 758                         rx_ring->rx_stats.alloc_rx_buff_failed++;
 759                         return NULL;
 760                 }
 761
 762                 /* we will be copying header into skb->data in
 763                  * pskb_may_pull so it is in our interest to prefetch
 764                  * it now to avoid a possible cache miss
 765                  */
 766                 prefetchw(skb->data);
 767         }
 768
 769         /* we are reusing so sync this buffer for CPU use */
 770         dma_sync_single_range_for_cpu(rx_ring->dev,
 771                                       rx_buffer->dma,
 772                                       rx_buffer->page_offset,
 773                                       IXGBEVF_RX_BUFSZ,
 774                                       DMA_FROM_DEVICE);
 775
 776         /* pull page into skb */
 777         if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
 778                 /* hand second half of page back to the ring */
 779                 ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
 780         } else {
 781                 /* we are not reusing the buffer so unmap it */
 782                 dma_unmap_page(rx_ring->dev, rx_buffer->dma,
 783                                PAGE_SIZE, DMA_FROM_DEVICE);
 784         }
 785
 786         /* clear contents of buffer_info */
 787         rx_buffer->dma = 0;
 788         rx_buffer->page = NULL;
 789
 790         return skb;
 791 }
 792
 793 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 794                                              u32 qmask)
 795 {
 796         struct ixgbe_hw *hw = &adapter->hw;
 797
 798         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 799 }
 800
 801 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 802                                 struct ixgbevf_ring *rx_ring,
 803                                 int budget)
 804 {
 805         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 806         u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
 807         struct sk_buff *skb = rx_ring->skb;
 808
 809         while (likely(total_rx_packets < budget)) {
 810                 union ixgbe_adv_rx_desc *rx_desc;
 811
 812                 /* return some buffers to hardware, one at a time is too slow */
 813                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 814                         ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
 815                         cleaned_count = 0;
 816                 }
 817
 818                 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
 819
 820                 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
 821                         break;
 822
 823                 /* This memory barrier is needed to keep us from reading
 824                  * any other fields out of the rx_desc until we know the
 825                  * RXD_STAT_DD bit is set
 826                  */
 827                 rmb();
 828
 829                 /* retrieve a buffer from the ring */
 830                 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);
 831
 832                 /* exit if we failed to retrieve a buffer */
 833                 if (!skb)
 834                         break;
 835
 836                 cleaned_count++;
 837
 838                 /* fetch next buffer in frame if non-eop */
 839                 if (ixgbevf_is_non_eop(rx_ring, rx_desc))
 840                         continue;
 841
 842                 /* verify the packet layout is correct */
 843                 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
 844                         skb = NULL;
 845                         continue;
 846                 }
 847
 848                 /* probably a little skewed due to removing CRC */
 849                 total_rx_bytes += skb->len;
 850
 851                 /* Workaround hardware that can't do proper VEPA multicast
 852                  * source pruning.
 853                  */
 854                 if ((skb->pkt_type == PACKET_BROADCAST ||
 855                     skb->pkt_type == PACKET_MULTICAST) &&
 856                     ether_addr_equal(rx_ring->netdev->dev_addr,
 857                                      eth_hdr(skb)->h_source)) {
 858                         dev_kfree_skb_irq(skb);
 859                         continue;
 860                 }
 861
 862                 /* populate checksum, VLAN, and protocol */
 863                 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
 864
 865                 ixgbevf_rx_skb(q_vector, skb);
 866
 867                 /* reset skb pointer */
 868                 skb = NULL;
 869
 870                 /* update budget accounting */
 871                 total_rx_packets++;
 872         }
 873
 874         /* place incomplete frames back on ring for completion */
 875         rx_ring->skb = skb;
 876
 877         u64_stats_update_begin(&rx_ring->syncp);
 878         rx_ring->stats.packets += total_rx_packets;
 879         rx_ring->stats.bytes += total_rx_bytes;
 880         u64_stats_update_end(&rx_ring->syncp);
 881         q_vector->rx.total_packets += total_rx_packets;
 882         q_vector->rx.total_bytes += total_rx_bytes;
 883
 884         return total_rx_packets;
 885 }
 886
 887 /**
 888  * ixgbevf_poll - NAPI polling calback
 889  * @napi: napi struct with our devices info in it
 890  * @budget: amount of work driver is allowed to do this pass, in packets
 891  *
 892  * This function will clean more than one or more rings associated with a
 893  * q_vector.
 894  **/
 895 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 896 {
 897         struct ixgbevf_q_vector *q_vector =
 898                 container_of(napi, struct ixgbevf_q_vector, napi);
 899         struct ixgbevf_adapter *adapter = q_vector->adapter;
 900         struct ixgbevf_ring *ring;
 901         int per_ring_budget;
 902         bool clean_complete = true;
 903
 904         ixgbevf_for_each_ring(ring, q_vector->tx)
 905                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 906
 907 #ifdef CONFIG_NET_RX_BUSY_POLL
 908         if (!ixgbevf_qv_lock_napi(q_vector))
 909                 return budget;
 910 #endif
 911
 912         /* attempt to distribute budget to each queue fairly, but don't allow
 913          * the budget to go below 1 because we'll exit polling */
 914         if (q_vector->rx.count > 1)
 915                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 916         else
 917                 per_ring_budget = budget;
 918
 919         ixgbevf_for_each_ring(ring, q_vector->rx)
 920                 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
 921                                                         per_ring_budget)
 922                                    < per_ring_budget);
 923
 924 #ifdef CONFIG_NET_RX_BUSY_POLL
 925         ixgbevf_qv_unlock_napi(q_vector);
 926 #endif
 927
 928         /* If all work not completed, return budget and keep polling */
 929         if (!clean_complete)
 930                 return budget;
 931         /* all work done, exit the polling mode */
 932         napi_complete(napi);
 933         if (adapter->rx_itr_setting & 1)
 934                 ixgbevf_set_itr(q_vector);
 935         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
 936             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
 937                 ixgbevf_irq_enable_queues(adapter,
 938                                           1 << q_vector->v_idx);
 939
 940         return 0;
 941 }
 942
 943 /**
 944  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 945  * @q_vector: structure containing interrupt and ring information
 946  */
 947 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 948 {
 949         struct ixgbevf_adapter *adapter = q_vector->adapter;
 950         struct ixgbe_hw *hw = &adapter->hw;
 951         int v_idx = q_vector->v_idx;
 952         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 953
 954         /*
 955          * set the WDIS bit to not clear the timer bits and cause an
 956          * immediate assertion of the interrupt
 957          */
 958         itr_reg |= IXGBE_EITR_CNT_WDIS;
 959
 960         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 961 }
 962
 963 #ifdef CONFIG_NET_RX_BUSY_POLL
 964 /* must be called with local_bh_disable()d */
 965 static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
 966 {
 967         struct ixgbevf_q_vector *q_vector =
 968                         container_of(napi, struct ixgbevf_q_vector, napi);
 969         struct ixgbevf_adapter *adapter = q_vector->adapter;
 970         struct ixgbevf_ring  *ring;
 971         int found = 0;
 972
 973         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 974                 return LL_FLUSH_FAILED;
 975
 976         if (!ixgbevf_qv_lock_poll(q_vector))
 977                 return LL_FLUSH_BUSY;
 978
 979         ixgbevf_for_each_ring(ring, q_vector->rx) {
 980                 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
 981 #ifdef BP_EXTENDED_STATS
 982                 if (found)
 983                         ring->stats.cleaned += found;
 984                 else
 985                         ring->stats.misses++;
 986 #endif
 987                 if (found)
 988                         break;
 989         }
 990
 991         ixgbevf_qv_unlock_poll(q_vector);
 992
 993         return found;
 994 }
 995 #endif /* CONFIG_NET_RX_BUSY_POLL */
 996
 997 /**
 998  * ixgbevf_configure_msix - Configure MSI-X hardware
 999  * @adapter: board private structure
1000  *
1001  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
1002  * interrupts.
1003  **/
1004 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
1005 {
1006         struct ixgbevf_q_vector *q_vector;
1007         int q_vectors, v_idx;
1008
1009         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1010         adapter->eims_enable_mask = 0;
1011
1012         /*
1013          * Populate the IVAR table and set the ITR values to the
1014          * corresponding register.
1015          */
1016         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1017                 struct ixgbevf_ring *ring;
1018                 q_vector = adapter->q_vector[v_idx];
1019
1020                 ixgbevf_for_each_ring(ring, q_vector->rx)
1021                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1022
1023                 ixgbevf_for_each_ring(ring, q_vector->tx)
1024                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1025
1026                 if (q_vector->tx.ring && !q_vector->rx.ring) {
1027                         /* tx only vector */
1028                         if (adapter->tx_itr_setting == 1)
1029                                 q_vector->itr = IXGBE_10K_ITR;
1030                         else
1031                                 q_vector->itr = adapter->tx_itr_setting;
1032                 } else {
1033                         /* rx or rx/tx vector */
1034                         if (adapter->rx_itr_setting == 1)
1035                                 q_vector->itr = IXGBE_20K_ITR;
1036                         else
1037                                 q_vector->itr = adapter->rx_itr_setting;
1038                 }
1039
1040                 /* add q_vector eims value to global eims_enable_mask */
1041                 adapter->eims_enable_mask |= 1 << v_idx;
1042
1043                 ixgbevf_write_eitr(q_vector);
1044         }
1045
1046         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
1047         /* setup eims_other and add value to global eims_enable_mask */
1048         adapter->eims_other = 1 << v_idx;
1049         adapter->eims_enable_mask |= adapter->eims_other;
1050 }
1051
1052 enum latency_range {
1053         lowest_latency = 0,
1054         low_latency = 1,
1055         bulk_latency = 2,
1056         latency_invalid = 255
1057 };
1058
1059 /**
1060  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
1061  * @q_vector: structure containing interrupt and ring information
1062  * @ring_container: structure containing ring performance data
1063  *
1064  *      Stores a new ITR value based on packets and byte
1065  *      counts during the last interrupt.  The advantage of per interrupt
1066  *      computation is faster updates and more accurate ITR for the current
1067  *      traffic pattern.  Constants in this function were computed
1068  *      based on theoretical maximum wire speed and thresholds were set based
1069  *      on testing data as well as attempting to minimize response time
1070  *      while increasing bulk throughput.
1071  **/
1072 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
1073                                struct ixgbevf_ring_container *ring_container)
1074 {
1075         int bytes = ring_container->total_bytes;
1076         int packets = ring_container->total_packets;
1077         u32 timepassed_us;
1078         u64 bytes_perint;
1079         u8 itr_setting = ring_container->itr;
1080
1081         if (packets == 0)
1082                 return;
1083
1084         /* simple throttlerate management
1085          *    0-20MB/s lowest (100000 ints/s)
1086          *   20-100MB/s low   (20000 ints/s)
1087          *  100-1249MB/s bulk (8000 ints/s)
1088          */
1089         /* what was last interrupt timeslice? */
1090         timepassed_us = q_vector->itr >> 2;
1091         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1092
1093         switch (itr_setting) {
1094         case lowest_latency:
1095                 if (bytes_perint > 10)
1096                         itr_setting = low_latency;
1097                 break;
1098         case low_latency:
1099                 if (bytes_perint > 20)
1100                         itr_setting = bulk_latency;
1101                 else if (bytes_perint <= 10)
1102                         itr_setting = lowest_latency;
1103                 break;
1104         case bulk_latency:
1105                 if (bytes_perint <= 20)
1106                         itr_setting = low_latency;
1107                 break;
1108         }
1109
1110         /* clear work counters since we have the values we need */
1111         ring_container->total_bytes = 0;
1112         ring_container->total_packets = 0;
1113
1114         /* write updated itr to ring container */
1115         ring_container->itr = itr_setting;
1116 }
1117
1118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1119 {
1120         u32 new_itr = q_vector->itr;
1121         u8 current_itr;
1122
1123         ixgbevf_update_itr(q_vector, &q_vector->tx);
1124         ixgbevf_update_itr(q_vector, &q_vector->rx);
1125
1126         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1127
1128         switch (current_itr) {
1129         /* counts and packets in update_itr are dependent on these numbers */
1130         case lowest_latency:
1131                 new_itr = IXGBE_100K_ITR;
1132                 break;
1133         case low_latency:
1134                 new_itr = IXGBE_20K_ITR;
1135                 break;
1136         case bulk_latency:
1137         default:
1138                 new_itr = IXGBE_8K_ITR;
1139                 break;
1140         }
1141
1142         if (new_itr != q_vector->itr) {
1143                 /* do an exponential smoothing */
1144                 new_itr = (10 * new_itr * q_vector->itr) /
1145                           ((9 * new_itr) + q_vector->itr);
1146
1147                 /* save the algorithm value here */
1148                 q_vector->itr = new_itr;
1149
1150                 ixgbevf_write_eitr(q_vector);
1151         }
1152 }
1153
1154 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1155 {
1156         struct ixgbevf_adapter *adapter = data;
1157         struct ixgbe_hw *hw = &adapter->hw;
1158
1159         hw->mac.get_link_status = 1;
1160
1161         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
1162             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
1163                 mod_timer(&adapter->watchdog_timer, jiffies);
1164
1165         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
1166
1167         return IRQ_HANDLED;
1168 }
1169
1170 /**
1171  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
1172  * @irq: unused
1173  * @data: pointer to our q_vector struct for this interrupt vector
1174  **/
1175 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
1176 {
1177         struct ixgbevf_q_vector *q_vector = data;
1178
1179         /* EIAM disabled interrupts (on this vector) for us */
1180         if (q_vector->rx.ring || q_vector->tx.ring)
1181                 napi_schedule(&q_vector->napi);
1182
1183         return IRQ_HANDLED;
1184 }
1185
1186 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
1187                                      int r_idx)
1188 {
1189         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1190
1191         a->rx_ring[r_idx]->next = q_vector->rx.ring;
1192         q_vector->rx.ring = a->rx_ring[r_idx];
1193         q_vector->rx.count++;
1194 }
1195
1196 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
1197                                      int t_idx)
1198 {
1199         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1200
1201         a->tx_ring[t_idx]->next = q_vector->tx.ring;
1202         q_vector->tx.ring = a->tx_ring[t_idx];
1203         q_vector->tx.count++;
1204 }
1205
1206 /**
1207  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
1208  * @adapter: board private structure to initialize
1209  *
1210  * This function maps descriptor rings to the queue-specific vectors
1211  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
1212  * one vector per ring/queue, but on a constrained vector budget, we
1213  * group the rings as "efficiently" as possible.  You would add new
1214  * mapping configurations in here.
1215  **/
1216 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
1217 {
1218         int q_vectors;
1219         int v_start = 0;
1220         int rxr_idx = 0, txr_idx = 0;
1221         int rxr_remaining = adapter->num_rx_queues;
1222         int txr_remaining = adapter->num_tx_queues;
1223         int i, j;
1224         int rqpv, tqpv;
1225         int err = 0;
1226
1227         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1228
1229         /*
1230          * The ideal configuration...
1231          * We have enough vectors to map one per queue.
1232          */
1233         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
1234                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
1235                         map_vector_to_rxq(adapter, v_start, rxr_idx);
1236
1237                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
1238                         map_vector_to_txq(adapter, v_start, txr_idx);
1239                 goto out;
1240         }
1241
1242         /*
1243          * If we don't have enough vectors for a 1-to-1
1244          * mapping, we'll have to group them so there are
1245          * multiple queues per vector.
1246          */
1247         /* Re-adjusting *qpv takes care of the remainder. */
1248         for (i = v_start; i < q_vectors; i++) {
1249                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
1250                 for (j = 0; j < rqpv; j++) {
1251                         map_vector_to_rxq(adapter, i, rxr_idx);
1252                         rxr_idx++;
1253                         rxr_remaining--;
1254                 }
1255         }
1256         for (i = v_start; i < q_vectors; i++) {
1257                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
1258                 for (j = 0; j < tqpv; j++) {
1259                         map_vector_to_txq(adapter, i, txr_idx);
1260                         txr_idx++;
1261                         txr_remaining--;
1262                 }
1263         }
1264
1265 out:
1266         return err;
1267 }
1268
1269 /**
1270  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1271  * @adapter: board private structure
1272  *
1273  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1274  * interrupts from the kernel.
1275  **/
1276 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1277 {
1278         struct net_device *netdev = adapter->netdev;
1279         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1280         int vector, err;
1281         int ri = 0, ti = 0;
1282
1283         for (vector = 0; vector < q_vectors; vector++) {
1284                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1285                 struct msix_entry *entry = &adapter->msix_entries[vector];
1286
1287                 if (q_vector->tx.ring && q_vector->rx.ring) {
1288                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1289                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
1290                         ti++;
1291                 } else if (q_vector->rx.ring) {
1292                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1293                                  "%s-%s-%d", netdev->name, "rx", ri++);
1294                 } else if (q_vector->tx.ring) {
1295                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1296                                  "%s-%s-%d", netdev->name, "tx", ti++);
1297                 } else {
1298                         /* skip this unused q_vector */
1299                         continue;
1300                 }
1301                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1302                                   q_vector->name, q_vector);
1303                 if (err) {
1304                         hw_dbg(&adapter->hw,
1305                                "request_irq failed for MSIX interrupt "
1306                                "Error: %d\n", err);
1307                         goto free_queue_irqs;
1308                 }
1309         }
1310
1311         err = request_irq(adapter->msix_entries[vector].vector,
1312                           &ixgbevf_msix_other, 0, netdev->name, adapter);
1313         if (err) {
1314                 hw_dbg(&adapter->hw,
1315                        "request_irq for msix_other failed: %d\n", err);
1316                 goto free_queue_irqs;
1317         }
1318
1319         return 0;
1320
1321 free_queue_irqs:
1322         while (vector) {
1323                 vector--;
1324                 free_irq(adapter->msix_entries[vector].vector,
1325                          adapter->q_vector[vector]);
1326         }
1327         /* This failure is non-recoverable - it indicates the system is
1328          * out of MSIX vector resources and the VF driver cannot run
1329          * without them.  Set the number of msix vectors to zero
1330          * indicating that not enough can be allocated.  The error
1331          * will be returned to the user indicating device open failed.
1332          * Any further attempts to force the driver to open will also
1333          * fail.  The only way to recover is to unload the driver and
1334          * reload it again.  If the system has recovered some MSIX
1335          * vectors then it may succeed.
1336          */
1337         adapter->num_msix_vectors = 0;
1338         return err;
1339 }
1340
1341 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1342 {
1343         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1344
1345         for (i = 0; i < q_vectors; i++) {
1346                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1347                 q_vector->rx.ring = NULL;
1348                 q_vector->tx.ring = NULL;
1349                 q_vector->rx.count = 0;
1350                 q_vector->tx.count = 0;
1351         }
1352 }
1353
1354 /**
1355  * ixgbevf_request_irq - initialize interrupts
1356  * @adapter: board private structure
1357  *
1358  * Attempts to configure interrupts using the best available
1359  * capabilities of the hardware and kernel.
1360  **/
1361 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1362 {
1363         int err = 0;
1364
1365         err = ixgbevf_request_msix_irqs(adapter);
1366
1367         if (err)
1368                 hw_dbg(&adapter->hw,
1369                        "request_irq failed, Error %d\n", err);
1370
1371         return err;
1372 }
1373
1374 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1375 {
1376         int i, q_vectors;
1377
1378         q_vectors = adapter->num_msix_vectors;
1379         i = q_vectors - 1;
1380
1381         free_irq(adapter->msix_entries[i].vector, adapter);
1382         i--;
1383
1384         for (; i >= 0; i--) {
1385                 /* free only the irqs that were actually requested */
1386                 if (!adapter->q_vector[i]->rx.ring &&
1387                     !adapter->q_vector[i]->tx.ring)
1388                         continue;
1389
1390                 free_irq(adapter->msix_entries[i].vector,
1391                          adapter->q_vector[i]);
1392         }
1393
1394         ixgbevf_reset_q_vectors(adapter);
1395 }
1396
1397 /**
1398  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1399  * @adapter: board private structure
1400  **/
1401 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1402 {
1403         struct ixgbe_hw *hw = &adapter->hw;
1404         int i;
1405
1406         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1407         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1408         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1409
1410         IXGBE_WRITE_FLUSH(hw);
1411
1412         for (i = 0; i < adapter->num_msix_vectors; i++)
1413                 synchronize_irq(adapter->msix_entries[i].vector);
1414 }
1415
1416 /**
1417  * ixgbevf_irq_enable - Enable default interrupt generation settings
1418  * @adapter: board private structure
1419  **/
1420 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1421 {
1422         struct ixgbe_hw *hw = &adapter->hw;
1423
1424         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1425         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1426         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1427 }
1428
1429 /**
1430  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1431  * @adapter: board private structure
1432  * @ring: structure containing ring specific data
1433  *
1434  * Configure the Tx descriptor ring after a reset.
1435  **/
1436 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1437                                       struct ixgbevf_ring *ring)
1438 {
1439         struct ixgbe_hw *hw = &adapter->hw;
1440         u64 tdba = ring->dma;
1441         int wait_loop = 10;
1442         u32 txdctl = IXGBE_TXDCTL_ENABLE;
1443         u8 reg_idx = ring->reg_idx;
1444
1445         /* disable queue to avoid issues while updating state */
1446         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1447         IXGBE_WRITE_FLUSH(hw);
1448
1449         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1450         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1451         IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1452                         ring->count * sizeof(union ixgbe_adv_tx_desc));
1453
1454         /* disable head writeback */
1455         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1456         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1457
1458         /* enable relaxed ordering */
1459         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1460                         (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1461                          IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1462
1463         /* reset head and tail pointers */
1464         IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1465         IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1466         ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
1467
1468         /* reset ntu and ntc to place SW in sync with hardwdare */
1469         ring->next_to_clean = 0;
1470         ring->next_to_use = 0;
1471
1472         /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1473          * to or less than the number of on chip descriptors, which is
1474          * currently 40.
1475          */
1476         txdctl |= (8 << 16);    /* WTHRESH = 8 */
1477
1478         /* Setting PTHRESH to 32 both improves performance */
1479         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
1480                   32;          /* PTHRESH = 32 */
1481
1482         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1483
1484         /* poll to verify queue is enabled */
1485         do {
1486                 usleep_range(1000, 2000);
1487                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1488         }  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1489         if (!wait_loop)
1490                 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1491 }
1492
1493 /**
1494  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1495  * @adapter: board private structure
1496  *
1497  * Configure the Tx unit of the MAC after a reset.
1498  **/
1499 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1500 {
1501         u32 i;
1502
1503         /* Setup the HW Tx Head and Tail descriptor pointers */
1504         for (i = 0; i < adapter->num_tx_queues; i++)
1505                 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
1506 }
1507
1508 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1509
1510 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1511 {
1512         struct ixgbe_hw *hw = &adapter->hw;
1513         u32 srrctl;
1514
1515         srrctl = IXGBE_SRRCTL_DROP_EN;
1516
1517         srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
1518         srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1519         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1520
1521         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1522 }
1523
1524 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1525 {
1526         struct ixgbe_hw *hw = &adapter->hw;
1527
1528         /* PSRTYPE must be initialized in 82599 */
1529         u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1530                       IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1531                       IXGBE_PSRTYPE_L2HDR;
1532
1533         if (adapter->num_rx_queues > 1)
1534                 psrtype |= 1 << 29;
1535
1536         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1537 }
1538
1539 #define IXGBEVF_MAX_RX_DESC_POLL 10
1540 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1541                                      struct ixgbevf_ring *ring)
1542 {
1543         struct ixgbe_hw *hw = &adapter->hw;
1544         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1545         u32 rxdctl;
1546         u8 reg_idx = ring->reg_idx;
1547
1548         if (IXGBE_REMOVED(hw->hw_addr))
1549                 return;
1550         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1551         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1552
1553         /* write value back with RXDCTL.ENABLE bit cleared */
1554         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1555
1556         /* the hardware may take up to 100us to really disable the rx queue */
1557         do {
1558                 udelay(10);
1559                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1560         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1561
1562         if (!wait_loop)
1563                 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1564                        reg_idx);
1565 }
1566
1567 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1568                                          struct ixgbevf_ring *ring)
1569 {
1570         struct ixgbe_hw *hw = &adapter->hw;
1571         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1572         u32 rxdctl;
1573         u8 reg_idx = ring->reg_idx;
1574
1575         if (IXGBE_REMOVED(hw->hw_addr))
1576                 return;
1577         do {
1578                 usleep_range(1000, 2000);
1579                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1580         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1581
1582         if (!wait_loop)
1583                 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1584                        reg_idx);
1585 }
1586
1587 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter)
1588 {
1589         struct ixgbe_hw *hw = &adapter->hw;
1590         u32 vfmrqc = 0, vfreta = 0;
1591         u32 rss_key[10];
1592         u16 rss_i = adapter->num_rx_queues;
1593         int i, j;
1594
1595         /* Fill out hash function seeds */
1596         netdev_rss_key_fill(rss_key, sizeof(rss_key));
1597         for (i = 0; i < 10; i++)
1598                 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]);
1599
1600         /* Fill out redirection table */
1601         for (i = 0, j = 0; i < 64; i++, j++) {
1602                 if (j == rss_i)
1603                         j = 0;
1604                 vfreta = (vfreta << 8) | (j * 0x1);
1605                 if ((i & 3) == 3)
1606                         IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta);
1607         }
1608
1609         /* Perform hash on these packet types */
1610         vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 |
1611                 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP |
1612                 IXGBE_VFMRQC_RSS_FIELD_IPV6 |
1613                 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP;
1614
1615         vfmrqc |= IXGBE_VFMRQC_RSSEN;
1616
1617         IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc);
1618 }
1619
1620 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1621                                       struct ixgbevf_ring *ring)
1622 {
1623         struct ixgbe_hw *hw = &adapter->hw;
1624         u64 rdba = ring->dma;
1625         u32 rxdctl;
1626         u8 reg_idx = ring->reg_idx;
1627
1628         /* disable queue to avoid issues while updating state */
1629         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1630         ixgbevf_disable_rx_queue(adapter, ring);
1631
1632         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1633         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1634         IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1635                         ring->count * sizeof(union ixgbe_adv_rx_desc));
1636
1637         /* enable relaxed ordering */
1638         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1639                         IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1640
1641         /* reset head and tail pointers */
1642         IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1643         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1644         ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
1645
1646         /* reset ntu and ntc to place SW in sync with hardwdare */
1647         ring->next_to_clean = 0;
1648         ring->next_to_use = 0;
1649         ring->next_to_alloc = 0;
1650
1651         ixgbevf_configure_srrctl(adapter, reg_idx);
1652
1653         /* allow any size packet since we can handle overflow */
1654         rxdctl &= ~IXGBE_RXDCTL_RLPML_EN;
1655
1656         rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1657         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1658
1659         ixgbevf_rx_desc_queue_enable(adapter, ring);
1660         ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
1661 }
1662
1663 /**
1664  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1665  * @adapter: board private structure
1666  *
1667  * Configure the Rx unit of the MAC after a reset.
1668  **/
1669 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1670 {
1671         int i;
1672         struct ixgbe_hw *hw = &adapter->hw;
1673         struct net_device *netdev = adapter->netdev;
1674
1675         ixgbevf_setup_psrtype(adapter);
1676         if (hw->mac.type >= ixgbe_mac_X550_vf)
1677                 ixgbevf_setup_vfmrqc(adapter);
1678
1679         /* notify the PF of our intent to use this size of frame */
1680         ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
1681
1682         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1683          * the Base and Length of the Rx Descriptor Ring */
1684         for (i = 0; i < adapter->num_rx_queues; i++)
1685                 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
1686 }
1687
1688 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1689                                    __be16 proto, u16 vid)
1690 {
1691         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1692         struct ixgbe_hw *hw = &adapter->hw;
1693         int err;
1694
1695         spin_lock_bh(&adapter->mbx_lock);
1696
1697         /* add VID to filter table */
1698         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1699
1700         spin_unlock_bh(&adapter->mbx_lock);
1701
1702         /* translate error return types so error makes sense */
1703         if (err == IXGBE_ERR_MBX)
1704                 return -EIO;
1705
1706         if (err == IXGBE_ERR_INVALID_ARGUMENT)
1707                 return -EACCES;
1708
1709         set_bit(vid, adapter->active_vlans);
1710
1711         return err;
1712 }
1713
1714 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1715                                     __be16 proto, u16 vid)
1716 {
1717         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1718         struct ixgbe_hw *hw = &adapter->hw;
1719         int err = -EOPNOTSUPP;
1720
1721         spin_lock_bh(&adapter->mbx_lock);
1722
1723         /* remove VID from filter table */
1724         err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1725
1726         spin_unlock_bh(&adapter->mbx_lock);
1727
1728         clear_bit(vid, adapter->active_vlans);
1729
1730         return err;
1731 }
1732
1733 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1734 {
1735         u16 vid;
1736
1737         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1738                 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1739                                         htons(ETH_P_8021Q), vid);
1740 }
1741
1742 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1743 {
1744         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1745         struct ixgbe_hw *hw = &adapter->hw;
1746         int count = 0;
1747
1748         if ((netdev_uc_count(netdev)) > 10) {
1749                 pr_err("Too many unicast filters - No Space\n");
1750                 return -ENOSPC;
1751         }
1752
1753         if (!netdev_uc_empty(netdev)) {
1754                 struct netdev_hw_addr *ha;
1755                 netdev_for_each_uc_addr(ha, netdev) {
1756                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1757                         udelay(200);
1758                 }
1759         } else {
1760                 /*
1761                  * If the list is empty then send message to PF driver to
1762                  * clear all macvlans on this VF.
1763                  */
1764                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1765         }
1766
1767         return count;
1768 }
1769
1770 /**
1771  * ixgbevf_set_rx_mode - Multicast and unicast set
1772  * @netdev: network interface device structure
1773  *
1774  * The set_rx_method entry point is called whenever the multicast address
1775  * list, unicast address list or the network interface flags are updated.
1776  * This routine is responsible for configuring the hardware for proper
1777  * multicast mode and configuring requested unicast filters.
1778  **/
1779 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1780 {
1781         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1782         struct ixgbe_hw *hw = &adapter->hw;
1783
1784         spin_lock_bh(&adapter->mbx_lock);
1785
1786         /* reprogram multicast list */
1787         hw->mac.ops.update_mc_addr_list(hw, netdev);
1788
1789         ixgbevf_write_uc_addr_list(netdev);
1790
1791         spin_unlock_bh(&adapter->mbx_lock);
1792 }
1793
1794 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1795 {
1796         int q_idx;
1797         struct ixgbevf_q_vector *q_vector;
1798         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1799
1800         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1801                 q_vector = adapter->q_vector[q_idx];
1802 #ifdef CONFIG_NET_RX_BUSY_POLL
1803                 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1804 #endif
1805                 napi_enable(&q_vector->napi);
1806         }
1807 }
1808
1809 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1810 {
1811         int q_idx;
1812         struct ixgbevf_q_vector *q_vector;
1813         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1814
1815         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1816                 q_vector = adapter->q_vector[q_idx];
1817                 napi_disable(&q_vector->napi);
1818 #ifdef CONFIG_NET_RX_BUSY_POLL
1819                 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1820                         pr_info("QV %d locked\n", q_idx);
1821                         usleep_range(1000, 20000);
1822                 }
1823 #endif /* CONFIG_NET_RX_BUSY_POLL */
1824         }
1825 }
1826
1827 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1828 {
1829         struct ixgbe_hw *hw = &adapter->hw;
1830         unsigned int def_q = 0;
1831         unsigned int num_tcs = 0;
1832         unsigned int num_rx_queues = adapter->num_rx_queues;
1833         unsigned int num_tx_queues = adapter->num_tx_queues;
1834         int err;
1835
1836         spin_lock_bh(&adapter->mbx_lock);
1837
1838         /* fetch queue configuration from the PF */
1839         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1840
1841         spin_unlock_bh(&adapter->mbx_lock);
1842
1843         if (err)
1844                 return err;
1845
1846         if (num_tcs > 1) {
1847                 /* we need only one Tx queue */
1848                 num_tx_queues = 1;
1849
1850                 /* update default Tx ring register index */
1851                 adapter->tx_ring[0]->reg_idx = def_q;
1852
1853                 /* we need as many queues as traffic classes */
1854                 num_rx_queues = num_tcs;
1855         }
1856
1857         /* if we have a bad config abort request queue reset */
1858         if ((adapter->num_rx_queues != num_rx_queues) ||
1859             (adapter->num_tx_queues != num_tx_queues)) {
1860                 /* force mailbox timeout to prevent further messages */
1861                 hw->mbx.timeout = 0;
1862
1863                 /* wait for watchdog to come around and bail us out */
1864                 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1865         }
1866
1867         return 0;
1868 }
1869
1870 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1871 {
1872         ixgbevf_configure_dcb(adapter);
1873
1874         ixgbevf_set_rx_mode(adapter->netdev);
1875
1876         ixgbevf_restore_vlan(adapter);
1877
1878         ixgbevf_configure_tx(adapter);
1879         ixgbevf_configure_rx(adapter);
1880 }
1881
1882 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1883 {
1884         /* Only save pre-reset stats if there are some */
1885         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1886                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1887                         adapter->stats.base_vfgprc;
1888                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1889                         adapter->stats.base_vfgptc;
1890                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1891                         adapter->stats.base_vfgorc;
1892                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1893                         adapter->stats.base_vfgotc;
1894                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1895                         adapter->stats.base_vfmprc;
1896         }
1897 }
1898
1899 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1900 {
1901         struct ixgbe_hw *hw = &adapter->hw;
1902
1903         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1904         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1905         adapter->stats.last_vfgorc |=
1906                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1907         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1908         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1909         adapter->stats.last_vfgotc |=
1910                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1911         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1912
1913         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1914         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1915         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1916         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1917         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1918 }
1919
1920 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1921 {
1922         struct ixgbe_hw *hw = &adapter->hw;
1923         int api[] = { ixgbe_mbox_api_11,
1924                       ixgbe_mbox_api_10,
1925                       ixgbe_mbox_api_unknown };
1926         int err = 0, idx = 0;
1927
1928         spin_lock_bh(&adapter->mbx_lock);
1929
1930         while (api[idx] != ixgbe_mbox_api_unknown) {
1931                 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1932                 if (!err)
1933                         break;
1934                 idx++;
1935         }
1936
1937         spin_unlock_bh(&adapter->mbx_lock);
1938 }
1939
1940 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1941 {
1942         struct net_device *netdev = adapter->netdev;
1943         struct ixgbe_hw *hw = &adapter->hw;
1944
1945         ixgbevf_configure_msix(adapter);
1946
1947         spin_lock_bh(&adapter->mbx_lock);
1948
1949         if (is_valid_ether_addr(hw->mac.addr))
1950                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1951         else
1952                 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1953
1954         spin_unlock_bh(&adapter->mbx_lock);
1955
1956         smp_mb__before_atomic();
1957         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1958         ixgbevf_napi_enable_all(adapter);
1959
1960         /* enable transmits */
1961         netif_tx_start_all_queues(netdev);
1962
1963         ixgbevf_save_reset_stats(adapter);
1964         ixgbevf_init_last_counter_stats(adapter);
1965
1966         hw->mac.get_link_status = 1;
1967         mod_timer(&adapter->watchdog_timer, jiffies);
1968 }
1969
1970 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1971 {
1972         struct ixgbe_hw *hw = &adapter->hw;
1973
1974         ixgbevf_configure(adapter);
1975
1976         ixgbevf_up_complete(adapter);
1977
1978         /* clear any pending interrupts, may auto mask */
1979         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1980
1981         ixgbevf_irq_enable(adapter);
1982 }
1983
1984 /**
1985  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1986  * @rx_ring: ring to free buffers from
1987  **/
1988 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
1989 {
1990         struct device *dev = rx_ring->dev;
1991         unsigned long size;
1992         unsigned int i;
1993
1994         /* Free Rx ring sk_buff */
1995         if (rx_ring->skb) {
1996                 dev_kfree_skb(rx_ring->skb);
1997                 rx_ring->skb = NULL;
1998         }
1999
2000         /* ring already cleared, nothing to do */
2001         if (!rx_ring->rx_buffer_info)
2002                 return;
2003
2004         /* Free all the Rx ring pages */
2005         for (i = 0; i < rx_ring->count; i++) {
2006                 struct ixgbevf_rx_buffer *rx_buffer;
2007
2008                 rx_buffer = &rx_ring->rx_buffer_info[i];
2009                 if (rx_buffer->dma)
2010                         dma_unmap_page(dev, rx_buffer->dma,
2011                                        PAGE_SIZE, DMA_FROM_DEVICE);
2012                 rx_buffer->dma = 0;
2013                 if (rx_buffer->page)
2014                         __free_page(rx_buffer->page);
2015                 rx_buffer->page = NULL;
2016         }
2017
2018         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2019         memset(rx_ring->rx_buffer_info, 0, size);
2020
2021         /* Zero out the descriptor ring */
2022         memset(rx_ring->desc, 0, rx_ring->size);
2023 }
2024
2025 /**
2026  * ixgbevf_clean_tx_ring - Free Tx Buffers
2027  * @tx_ring: ring to be cleaned
2028  **/
2029 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
2030 {
2031         struct ixgbevf_tx_buffer *tx_buffer_info;
2032         unsigned long size;
2033         unsigned int i;
2034
2035         if (!tx_ring->tx_buffer_info)
2036                 return;
2037
2038         /* Free all the Tx ring sk_buffs */
2039         for (i = 0; i < tx_ring->count; i++) {
2040                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2041                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2042         }
2043
2044         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2045         memset(tx_ring->tx_buffer_info, 0, size);
2046
2047         memset(tx_ring->desc, 0, tx_ring->size);
2048 }
2049
2050 /**
2051  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
2052  * @adapter: board private structure
2053  **/
2054 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
2055 {
2056         int i;
2057
2058         for (i = 0; i < adapter->num_rx_queues; i++)
2059                 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
2060 }
2061
2062 /**
2063  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
2064  * @adapter: board private structure
2065  **/
2066 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
2067 {
2068         int i;
2069
2070         for (i = 0; i < adapter->num_tx_queues; i++)
2071                 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
2072 }
2073
2074 void ixgbevf_down(struct ixgbevf_adapter *adapter)
2075 {
2076         struct net_device *netdev = adapter->netdev;
2077         struct ixgbe_hw *hw = &adapter->hw;
2078         int i;
2079
2080         /* signal that we are down to the interrupt handler */
2081         if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
2082                 return; /* do nothing if already down */
2083
2084         /* disable all enabled rx queues */
2085         for (i = 0; i < adapter->num_rx_queues; i++)
2086                 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
2087
2088         netif_tx_disable(netdev);
2089
2090         msleep(10);
2091
2092         netif_tx_stop_all_queues(netdev);
2093
2094         ixgbevf_irq_disable(adapter);
2095
2096         ixgbevf_napi_disable_all(adapter);
2097
2098         del_timer_sync(&adapter->watchdog_timer);
2099         /* can't call flush scheduled work here because it can deadlock
2100          * if linkwatch_event tries to acquire the rtnl_lock which we are
2101          * holding */
2102         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
2103                 msleep(1);
2104
2105         /* disable transmits in the hardware now that interrupts are off */
2106         for (i = 0; i < adapter->num_tx_queues; i++) {
2107                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
2108
2109                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2110                                 IXGBE_TXDCTL_SWFLSH);
2111         }
2112
2113         netif_carrier_off(netdev);
2114
2115         if (!pci_channel_offline(adapter->pdev))
2116                 ixgbevf_reset(adapter);
2117
2118         ixgbevf_clean_all_tx_rings(adapter);
2119         ixgbevf_clean_all_rx_rings(adapter);
2120 }
2121
2122 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
2123 {
2124         WARN_ON(in_interrupt());
2125
2126         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
2127                 msleep(1);
2128
2129         ixgbevf_down(adapter);
2130         ixgbevf_up(adapter);
2131
2132         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
2133 }
2134
2135 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
2136 {
2137         struct ixgbe_hw *hw = &adapter->hw;
2138         struct net_device *netdev = adapter->netdev;
2139
2140         if (hw->mac.ops.reset_hw(hw)) {
2141                 hw_dbg(hw, "PF still resetting\n");
2142         } else {
2143                 hw->mac.ops.init_hw(hw);
2144                 ixgbevf_negotiate_api(adapter);
2145         }
2146
2147         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
2148                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
2149                        netdev->addr_len);
2150                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
2151                        netdev->addr_len);
2152         }
2153 }
2154
2155 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
2156                                         int vectors)
2157 {
2158         int vector_threshold;
2159
2160         /* We'll want at least 2 (vector_threshold):
2161          * 1) TxQ[0] + RxQ[0] handler
2162          * 2) Other (Link Status Change, etc.)
2163          */
2164         vector_threshold = MIN_MSIX_COUNT;
2165
2166         /* The more we get, the more we will assign to Tx/Rx Cleanup
2167          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
2168          * Right now, we simply care about how many we'll get; we'll
2169          * set them up later while requesting irq's.
2170          */
2171         vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2172                                         vector_threshold, vectors);
2173
2174         if (vectors < 0) {
2175                 dev_err(&adapter->pdev->dev,
2176                         "Unable to allocate MSI-X interrupts\n");
2177                 kfree(adapter->msix_entries);
2178                 adapter->msix_entries = NULL;
2179                 return vectors;
2180         }
2181
2182         /* Adjust for only the vectors we'll use, which is minimum
2183          * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
2184          * vectors we were allocated.
2185          */
2186         adapter->num_msix_vectors = vectors;
2187
2188         return 0;
2189 }
2190
2191 /**
2192  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
2193  * @adapter: board private structure to initialize
2194  *
2195  * This is the top level queue allocation routine.  The order here is very
2196  * important, starting with the "most" number of features turned on at once,
2197  * and ending with the smallest set of features.  This way large combinations
2198  * can be allocated if they're turned on, and smaller combinations are the
2199  * fallthrough conditions.
2200  *
2201  **/
2202 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
2203 {
2204         struct ixgbe_hw *hw = &adapter->hw;
2205         unsigned int def_q = 0;
2206         unsigned int num_tcs = 0;
2207         int err;
2208
2209         /* Start with base case */
2210         adapter->num_rx_queues = 1;
2211         adapter->num_tx_queues = 1;
2212
2213         spin_lock_bh(&adapter->mbx_lock);
2214
2215         /* fetch queue configuration from the PF */
2216         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2217
2218         spin_unlock_bh(&adapter->mbx_lock);
2219
2220         if (err)
2221                 return;
2222
2223         /* we need as many queues as traffic classes */
2224         if (num_tcs > 1) {
2225                 adapter->num_rx_queues = num_tcs;
2226         } else {
2227                 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES);
2228
2229                 switch (hw->api_version) {
2230                 case ixgbe_mbox_api_11:
2231                         adapter->num_rx_queues = rss;
2232                         adapter->num_tx_queues = rss;
2233                 default:
2234                         break;
2235                 }
2236         }
2237 }
2238
2239 /**
2240  * ixgbevf_alloc_queues - Allocate memory for all rings
2241  * @adapter: board private structure to initialize
2242  *
2243  * We allocate one ring per queue at run-time since we don't know the
2244  * number of queues at compile-time.  The polling_netdev array is
2245  * intended for Multiqueue, but should work fine with a single queue.
2246  **/
2247 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
2248 {
2249         struct ixgbevf_ring *ring;
2250         int rx = 0, tx = 0;
2251
2252         for (; tx < adapter->num_tx_queues; tx++) {
2253                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2254                 if (!ring)
2255                         goto err_allocation;
2256
2257                 ring->dev = &adapter->pdev->dev;
2258                 ring->netdev = adapter->netdev;
2259                 ring->count = adapter->tx_ring_count;
2260                 ring->queue_index = tx;
2261                 ring->reg_idx = tx;
2262
2263                 adapter->tx_ring[tx] = ring;
2264         }
2265
2266         for (; rx < adapter->num_rx_queues; rx++) {
2267                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2268                 if (!ring)
2269                         goto err_allocation;
2270
2271                 ring->dev = &adapter->pdev->dev;
2272                 ring->netdev = adapter->netdev;
2273
2274                 ring->count = adapter->rx_ring_count;
2275                 ring->queue_index = rx;
2276                 ring->reg_idx = rx;
2277
2278                 adapter->rx_ring[rx] = ring;
2279         }
2280
2281         return 0;
2282
2283 err_allocation:
2284         while (tx) {
2285                 kfree(adapter->tx_ring[--tx]);
2286                 adapter->tx_ring[tx] = NULL;
2287         }
2288
2289         while (rx) {
2290                 kfree(adapter->rx_ring[--rx]);
2291                 adapter->rx_ring[rx] = NULL;
2292         }
2293         return -ENOMEM;
2294 }
2295
2296 /**
2297  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2298  * @adapter: board private structure to initialize
2299  *
2300  * Attempt to configure the interrupts using the best available
2301  * capabilities of the hardware and the kernel.
2302  **/
2303 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2304 {
2305         struct net_device *netdev = adapter->netdev;
2306         int err = 0;
2307         int vector, v_budget;
2308
2309         /*
2310          * It's easy to be greedy for MSI-X vectors, but it really
2311          * doesn't do us much good if we have a lot more vectors
2312          * than CPU's.  So let's be conservative and only ask for
2313          * (roughly) the same number of vectors as there are CPU's.
2314          * The default is to use pairs of vectors.
2315          */
2316         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2317         v_budget = min_t(int, v_budget, num_online_cpus());
2318         v_budget += NON_Q_VECTORS;
2319
2320         /* A failure in MSI-X entry allocation isn't fatal, but it does
2321          * mean we disable MSI-X capabilities of the adapter. */
2322         adapter->msix_entries = kcalloc(v_budget,
2323                                         sizeof(struct msix_entry), GFP_KERNEL);
2324         if (!adapter->msix_entries) {
2325                 err = -ENOMEM;
2326                 goto out;
2327         }
2328
2329         for (vector = 0; vector < v_budget; vector++)
2330                 adapter->msix_entries[vector].entry = vector;
2331
2332         err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
2333         if (err)
2334                 goto out;
2335
2336         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
2337         if (err)
2338                 goto out;
2339
2340         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
2341
2342 out:
2343         return err;
2344 }
2345
2346 /**
2347  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2348  * @adapter: board private structure to initialize
2349  *
2350  * We allocate one q_vector per queue interrupt.  If allocation fails we
2351  * return -ENOMEM.
2352  **/
2353 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2354 {
2355         int q_idx, num_q_vectors;
2356         struct ixgbevf_q_vector *q_vector;
2357
2358         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2359
2360         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2361                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2362                 if (!q_vector)
2363                         goto err_out;
2364                 q_vector->adapter = adapter;
2365                 q_vector->v_idx = q_idx;
2366                 netif_napi_add(adapter->netdev, &q_vector->napi,
2367                                ixgbevf_poll, 64);
2368 #ifdef CONFIG_NET_RX_BUSY_POLL
2369                 napi_hash_add(&q_vector->napi);
2370 #endif
2371                 adapter->q_vector[q_idx] = q_vector;
2372         }
2373
2374         return 0;
2375
2376 err_out:
2377         while (q_idx) {
2378                 q_idx--;
2379                 q_vector = adapter->q_vector[q_idx];
2380 #ifdef CONFIG_NET_RX_BUSY_POLL
2381                 napi_hash_del(&q_vector->napi);
2382 #endif
2383                 netif_napi_del(&q_vector->napi);
2384                 kfree(q_vector);
2385                 adapter->q_vector[q_idx] = NULL;
2386         }
2387         return -ENOMEM;
2388 }
2389
2390 /**
2391  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2392  * @adapter: board private structure to initialize
2393  *
2394  * This function frees the memory allocated to the q_vectors.  In addition if
2395  * NAPI is enabled it will delete any references to the NAPI struct prior
2396  * to freeing the q_vector.
2397  **/
2398 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2399 {
2400         int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2401
2402         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2403                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2404
2405                 adapter->q_vector[q_idx] = NULL;
2406 #ifdef CONFIG_NET_RX_BUSY_POLL
2407                 napi_hash_del(&q_vector->napi);
2408 #endif
2409                 netif_napi_del(&q_vector->napi);
2410                 kfree(q_vector);
2411         }
2412 }
2413
2414 /**
2415  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2416  * @adapter: board private structure
2417  *
2418  **/
2419 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2420 {
2421         pci_disable_msix(adapter->pdev);
2422         kfree(adapter->msix_entries);
2423         adapter->msix_entries = NULL;
2424 }
2425
2426 /**
2427  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2428  * @adapter: board private structure to initialize
2429  *
2430  **/
2431 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2432 {
2433         int err;
2434
2435         /* Number of supported queues */
2436         ixgbevf_set_num_queues(adapter);
2437
2438         err = ixgbevf_set_interrupt_capability(adapter);
2439         if (err) {
2440                 hw_dbg(&adapter->hw,
2441                        "Unable to setup interrupt capabilities\n");
2442                 goto err_set_interrupt;
2443         }
2444
2445         err = ixgbevf_alloc_q_vectors(adapter);
2446         if (err) {
2447                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2448                        "vectors\n");
2449                 goto err_alloc_q_vectors;
2450         }
2451
2452         err = ixgbevf_alloc_queues(adapter);
2453         if (err) {
2454                 pr_err("Unable to allocate memory for queues\n");
2455                 goto err_alloc_queues;
2456         }
2457
2458         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2459                "Tx Queue count = %u\n",
2460                (adapter->num_rx_queues > 1) ? "Enabled" :
2461                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2462
2463         set_bit(__IXGBEVF_DOWN, &adapter->state);
2464
2465         return 0;
2466 err_alloc_queues:
2467         ixgbevf_free_q_vectors(adapter);
2468 err_alloc_q_vectors:
2469         ixgbevf_reset_interrupt_capability(adapter);
2470 err_set_interrupt:
2471         return err;
2472 }
2473
2474 /**
2475  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2476  * @adapter: board private structure to clear interrupt scheme on
2477  *
2478  * We go through and clear interrupt specific resources and reset the structure
2479  * to pre-load conditions
2480  **/
2481 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2482 {
2483         int i;
2484
2485         for (i = 0; i < adapter->num_tx_queues; i++) {
2486                 kfree(adapter->tx_ring[i]);
2487                 adapter->tx_ring[i] = NULL;
2488         }
2489         for (i = 0; i < adapter->num_rx_queues; i++) {
2490                 kfree(adapter->rx_ring[i]);
2491                 adapter->rx_ring[i] = NULL;
2492         }
2493
2494         adapter->num_tx_queues = 0;
2495         adapter->num_rx_queues = 0;
2496
2497         ixgbevf_free_q_vectors(adapter);
2498         ixgbevf_reset_interrupt_capability(adapter);
2499 }
2500
2501 /**
2502  * ixgbevf_sw_init - Initialize general software structures
2503  * (struct ixgbevf_adapter)
2504  * @adapter: board private structure to initialize
2505  *
2506  * ixgbevf_sw_init initializes the Adapter private data structure.
2507  * Fields are initialized based on PCI device information and
2508  * OS network device settings (MTU size).
2509  **/
2510 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2511 {
2512         struct ixgbe_hw *hw = &adapter->hw;
2513         struct pci_dev *pdev = adapter->pdev;
2514         struct net_device *netdev = adapter->netdev;
2515         int err;
2516
2517         /* PCI config space info */
2518
2519         hw->vendor_id = pdev->vendor;
2520         hw->device_id = pdev->device;
2521         hw->revision_id = pdev->revision;
2522         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2523         hw->subsystem_device_id = pdev->subsystem_device;
2524
2525         hw->mbx.ops.init_params(hw);
2526
2527         /* assume legacy case in which PF would only give VF 2 queues */
2528         hw->mac.max_tx_queues = 2;
2529         hw->mac.max_rx_queues = 2;
2530
2531         /* lock to protect mailbox accesses */
2532         spin_lock_init(&adapter->mbx_lock);
2533
2534         err = hw->mac.ops.reset_hw(hw);
2535         if (err) {
2536                 dev_info(&pdev->dev,
2537                          "PF still in reset state.  Is the PF interface up?\n");
2538         } else {
2539                 err = hw->mac.ops.init_hw(hw);
2540                 if (err) {
2541                         pr_err("init_shared_code failed: %d\n", err);
2542                         goto out;
2543                 }
2544                 ixgbevf_negotiate_api(adapter);
2545                 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2546                 if (err)
2547                         dev_info(&pdev->dev, "Error reading MAC address\n");
2548                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2549                         dev_info(&pdev->dev,
2550                                  "MAC address not assigned by administrator.\n");
2551                 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2552         }
2553
2554         if (!is_valid_ether_addr(netdev->dev_addr)) {
2555                 dev_info(&pdev->dev, "Assigning random MAC address\n");
2556                 eth_hw_addr_random(netdev);
2557                 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
2558         }
2559
2560         /* Enable dynamic interrupt throttling rates */
2561         adapter->rx_itr_setting = 1;
2562         adapter->tx_itr_setting = 1;
2563
2564         /* set default ring sizes */
2565         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2566         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2567
2568         set_bit(__IXGBEVF_DOWN, &adapter->state);
2569         return 0;
2570
2571 out:
2572         return err;
2573 }
2574
2575 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
2576         {                                                       \
2577                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
2578                 if (current_counter < last_counter)             \
2579                         counter += 0x100000000LL;               \
2580                 last_counter = current_counter;                 \
2581                 counter &= 0xFFFFFFFF00000000LL;                \
2582                 counter |= current_counter;                     \
2583         }
2584
2585 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2586         {                                                                \
2587                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
2588                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
2589                 u64 current_counter = (current_counter_msb << 32) |      \
2590                         current_counter_lsb;                             \
2591                 if (current_counter < last_counter)                      \
2592                         counter += 0x1000000000LL;                       \
2593                 last_counter = current_counter;                          \
2594                 counter &= 0xFFFFFFF000000000LL;                         \
2595                 counter |= current_counter;                              \
2596         }
2597 /**
2598  * ixgbevf_update_stats - Update the board statistics counters.
2599  * @adapter: board private structure
2600  **/
2601 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2602 {
2603         struct ixgbe_hw *hw = &adapter->hw;
2604         int i;
2605
2606         if (!adapter->link_up)
2607                 return;
2608
2609         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2610                                 adapter->stats.vfgprc);
2611         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2612                                 adapter->stats.vfgptc);
2613         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2614                                 adapter->stats.last_vfgorc,
2615                                 adapter->stats.vfgorc);
2616         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2617                                 adapter->stats.last_vfgotc,
2618                                 adapter->stats.vfgotc);
2619         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2620                                 adapter->stats.vfmprc);
2621
2622         for (i = 0;  i  < adapter->num_rx_queues;  i++) {
2623                 adapter->hw_csum_rx_error +=
2624                         adapter->rx_ring[i]->hw_csum_rx_error;
2625                 adapter->rx_ring[i]->hw_csum_rx_error = 0;
2626         }
2627 }
2628
2629 /**
2630  * ixgbevf_watchdog - Timer Call-back
2631  * @data: pointer to adapter cast into an unsigned long
2632  **/
2633 static void ixgbevf_watchdog(unsigned long data)
2634 {
2635         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2636         struct ixgbe_hw *hw = &adapter->hw;
2637         u32 eics = 0;
2638         int i;
2639
2640         /*
2641          * Do the watchdog outside of interrupt context due to the lovely
2642          * delays that some of the newer hardware requires
2643          */
2644
2645         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2646                 goto watchdog_short_circuit;
2647
2648         /* get one bit for every active tx/rx interrupt vector */
2649         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2650                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2651                 if (qv->rx.ring || qv->tx.ring)
2652                         eics |= 1 << i;
2653         }
2654
2655         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2656
2657 watchdog_short_circuit:
2658         schedule_work(&adapter->watchdog_task);
2659 }
2660
2661 /**
2662  * ixgbevf_tx_timeout - Respond to a Tx Hang
2663  * @netdev: network interface device structure
2664  **/
2665 static void ixgbevf_tx_timeout(struct net_device *netdev)
2666 {
2667         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2668
2669         /* Do the reset outside of interrupt context */
2670         schedule_work(&adapter->reset_task);
2671 }
2672
2673 static void ixgbevf_reset_task(struct work_struct *work)
2674 {
2675         struct ixgbevf_adapter *adapter;
2676         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2677
2678         /* If we're already down or resetting, just bail */
2679         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2680             test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
2681             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2682                 return;
2683
2684         adapter->tx_timeout_count++;
2685
2686         ixgbevf_reinit_locked(adapter);
2687 }
2688
2689 /**
2690  * ixgbevf_watchdog_task - worker thread to bring link up
2691  * @work: pointer to work_struct containing our data
2692  **/
2693 static void ixgbevf_watchdog_task(struct work_struct *work)
2694 {
2695         struct ixgbevf_adapter *adapter = container_of(work,
2696                                                        struct ixgbevf_adapter,
2697                                                        watchdog_task);
2698         struct net_device *netdev = adapter->netdev;
2699         struct ixgbe_hw *hw = &adapter->hw;
2700         u32 link_speed = adapter->link_speed;
2701         bool link_up = adapter->link_up;
2702         s32 need_reset;
2703
2704         if (IXGBE_REMOVED(hw->hw_addr)) {
2705                 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
2706                         rtnl_lock();
2707                         ixgbevf_down(adapter);
2708                         rtnl_unlock();
2709                 }
2710                 return;
2711         }
2712         ixgbevf_queue_reset_subtask(adapter);
2713
2714         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2715
2716         /*
2717          * Always check the link on the watchdog because we have
2718          * no LSC interrupt
2719          */
2720         spin_lock_bh(&adapter->mbx_lock);
2721
2722         need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
2723
2724         spin_unlock_bh(&adapter->mbx_lock);
2725
2726         if (need_reset) {
2727                 adapter->link_up = link_up;
2728                 adapter->link_speed = link_speed;
2729                 netif_carrier_off(netdev);
2730                 netif_tx_stop_all_queues(netdev);
2731                 schedule_work(&adapter->reset_task);
2732                 goto pf_has_reset;
2733         }
2734         adapter->link_up = link_up;
2735         adapter->link_speed = link_speed;
2736
2737         if (link_up) {
2738                 if (!netif_carrier_ok(netdev)) {
2739                         char *link_speed_string;
2740                         switch (link_speed) {
2741                         case IXGBE_LINK_SPEED_10GB_FULL:
2742                                 link_speed_string = "10 Gbps";
2743                                 break;
2744                         case IXGBE_LINK_SPEED_1GB_FULL:
2745                                 link_speed_string = "1 Gbps";
2746                                 break;
2747                         case IXGBE_LINK_SPEED_100_FULL:
2748                                 link_speed_string = "100 Mbps";
2749                                 break;
2750                         default:
2751                                 link_speed_string = "unknown speed";
2752                                 break;
2753                         }
2754                         dev_info(&adapter->pdev->dev,
2755                                 "NIC Link is Up, %s\n", link_speed_string);
2756                         netif_carrier_on(netdev);
2757                         netif_tx_wake_all_queues(netdev);
2758                 }
2759         } else {
2760                 adapter->link_up = false;
2761                 adapter->link_speed = 0;
2762                 if (netif_carrier_ok(netdev)) {
2763                         dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
2764                         netif_carrier_off(netdev);
2765                         netif_tx_stop_all_queues(netdev);
2766                 }
2767         }
2768
2769         ixgbevf_update_stats(adapter);
2770
2771 pf_has_reset:
2772         /* Reset the timer */
2773         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
2774             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
2775                 mod_timer(&adapter->watchdog_timer,
2776                           round_jiffies(jiffies + (2 * HZ)));
2777
2778         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2779 }
2780
2781 /**
2782  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2783  * @tx_ring: Tx descriptor ring for a specific queue
2784  *
2785  * Free all transmit software resources
2786  **/
2787 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
2788 {
2789         ixgbevf_clean_tx_ring(tx_ring);
2790
2791         vfree(tx_ring->tx_buffer_info);
2792         tx_ring->tx_buffer_info = NULL;
2793
2794         /* if not set, then don't free */
2795         if (!tx_ring->desc)
2796                 return;
2797
2798         dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2799                           tx_ring->dma);
2800
2801         tx_ring->desc = NULL;
2802 }
2803
2804 /**
2805  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2806  * @adapter: board private structure
2807  *
2808  * Free all transmit software resources
2809  **/
2810 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2811 {
2812         int i;
2813
2814         for (i = 0; i < adapter->num_tx_queues; i++)
2815                 if (adapter->tx_ring[i]->desc)
2816                         ixgbevf_free_tx_resources(adapter->tx_ring[i]);
2817 }
2818
2819 /**
2820  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2821  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2822  *
2823  * Return 0 on success, negative on failure
2824  **/
2825 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
2826 {
2827         int size;
2828
2829         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2830         tx_ring->tx_buffer_info = vzalloc(size);
2831         if (!tx_ring->tx_buffer_info)
2832                 goto err;
2833
2834         /* round up to nearest 4K */
2835         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2836         tx_ring->size = ALIGN(tx_ring->size, 4096);
2837
2838         tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2839                                            &tx_ring->dma, GFP_KERNEL);
2840         if (!tx_ring->desc)
2841                 goto err;
2842
2843         return 0;
2844
2845 err:
2846         vfree(tx_ring->tx_buffer_info);
2847         tx_ring->tx_buffer_info = NULL;
2848         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2849                "descriptor ring\n");
2850         return -ENOMEM;
2851 }
2852
2853 /**
2854  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2855  * @adapter: board private structure
2856  *
2857  * If this function returns with an error, then it's possible one or
2858  * more of the rings is populated (while the rest are not).  It is the
2859  * callers duty to clean those orphaned rings.
2860  *
2861  * Return 0 on success, negative on failure
2862  **/
2863 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2864 {
2865         int i, err = 0;
2866
2867         for (i = 0; i < adapter->num_tx_queues; i++) {
2868                 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
2869                 if (!err)
2870                         continue;
2871                 hw_dbg(&adapter->hw,
2872                        "Allocation for Tx Queue %u failed\n", i);
2873                 break;
2874         }
2875
2876         return err;
2877 }
2878
2879 /**
2880  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2881  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2882  *
2883  * Returns 0 on success, negative on failure
2884  **/
2885 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
2886 {
2887         int size;
2888
2889         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2890         rx_ring->rx_buffer_info = vzalloc(size);
2891         if (!rx_ring->rx_buffer_info)
2892                 goto err;
2893
2894         /* Round up to nearest 4K */
2895         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2896         rx_ring->size = ALIGN(rx_ring->size, 4096);
2897
2898         rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2899                                            &rx_ring->dma, GFP_KERNEL);
2900
2901         if (!rx_ring->desc)
2902                 goto err;
2903
2904         return 0;
2905 err:
2906         vfree(rx_ring->rx_buffer_info);
2907         rx_ring->rx_buffer_info = NULL;
2908         dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
2909         return -ENOMEM;
2910 }
2911
2912 /**
2913  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2914  * @adapter: board private structure
2915  *
2916  * If this function returns with an error, then it's possible one or
2917  * more of the rings is populated (while the rest are not).  It is the
2918  * callers duty to clean those orphaned rings.
2919  *
2920  * Return 0 on success, negative on failure
2921  **/
2922 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2923 {
2924         int i, err = 0;
2925
2926         for (i = 0; i < adapter->num_rx_queues; i++) {
2927                 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
2928                 if (!err)
2929                         continue;
2930                 hw_dbg(&adapter->hw,
2931                        "Allocation for Rx Queue %u failed\n", i);
2932                 break;
2933         }
2934         return err;
2935 }
2936
2937 /**
2938  * ixgbevf_free_rx_resources - Free Rx Resources
2939  * @rx_ring: ring to clean the resources from
2940  *
2941  * Free all receive software resources
2942  **/
2943 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
2944 {
2945         ixgbevf_clean_rx_ring(rx_ring);
2946
2947         vfree(rx_ring->rx_buffer_info);
2948         rx_ring->rx_buffer_info = NULL;
2949
2950         dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2951                           rx_ring->dma);
2952
2953         rx_ring->desc = NULL;
2954 }
2955
2956 /**
2957  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2958  * @adapter: board private structure
2959  *
2960  * Free all receive software resources
2961  **/
2962 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2963 {
2964         int i;
2965
2966         for (i = 0; i < adapter->num_rx_queues; i++)
2967                 if (adapter->rx_ring[i]->desc)
2968                         ixgbevf_free_rx_resources(adapter->rx_ring[i]);
2969 }
2970
2971 /**
2972  * ixgbevf_open - Called when a network interface is made active
2973  * @netdev: network interface device structure
2974  *
2975  * Returns 0 on success, negative value on failure
2976  *
2977  * The open entry point is called when a network interface is made
2978  * active by the system (IFF_UP).  At this point all resources needed
2979  * for transmit and receive operations are allocated, the interrupt
2980  * handler is registered with the OS, the watchdog timer is started,
2981  * and the stack is notified that the interface is ready.
2982  **/
2983 static int ixgbevf_open(struct net_device *netdev)
2984 {
2985         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2986         struct ixgbe_hw *hw = &adapter->hw;
2987         int err;
2988
2989         /* A previous failure to open the device because of a lack of
2990          * available MSIX vector resources may have reset the number
2991          * of msix vectors variable to zero.  The only way to recover
2992          * is to unload/reload the driver and hope that the system has
2993          * been able to recover some MSIX vector resources.
2994          */
2995         if (!adapter->num_msix_vectors)
2996                 return -ENOMEM;
2997
2998         /* disallow open during test */
2999         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
3000                 return -EBUSY;
3001
3002         if (hw->adapter_stopped) {
3003                 ixgbevf_reset(adapter);
3004                 /* if adapter is still stopped then PF isn't up and
3005                  * the vf can't start. */
3006                 if (hw->adapter_stopped) {
3007                         err = IXGBE_ERR_MBX;
3008                         pr_err("Unable to start - perhaps the PF Driver isn't "
3009                                "up yet\n");
3010                         goto err_setup_reset;
3011                 }
3012         }
3013
3014         /* allocate transmit descriptors */
3015         err = ixgbevf_setup_all_tx_resources(adapter);
3016         if (err)
3017                 goto err_setup_tx;
3018
3019         /* allocate receive descriptors */
3020         err = ixgbevf_setup_all_rx_resources(adapter);
3021         if (err)
3022                 goto err_setup_rx;
3023
3024         ixgbevf_configure(adapter);
3025
3026         /*
3027          * Map the Tx/Rx rings to the vectors we were allotted.
3028          * if request_irq will be called in this function map_rings
3029          * must be called *before* up_complete
3030          */
3031         ixgbevf_map_rings_to_vectors(adapter);
3032
3033         ixgbevf_up_complete(adapter);
3034
3035         /* clear any pending interrupts, may auto mask */
3036         IXGBE_READ_REG(hw, IXGBE_VTEICR);
3037         err = ixgbevf_request_irq(adapter);
3038         if (err)
3039                 goto err_req_irq;
3040
3041         ixgbevf_irq_enable(adapter);
3042
3043         return 0;
3044
3045 err_req_irq:
3046         ixgbevf_down(adapter);
3047 err_setup_rx:
3048         ixgbevf_free_all_rx_resources(adapter);
3049 err_setup_tx:
3050         ixgbevf_free_all_tx_resources(adapter);
3051         ixgbevf_reset(adapter);
3052
3053 err_setup_reset:
3054
3055         return err;
3056 }
3057
3058 /**
3059  * ixgbevf_close - Disables a network interface
3060  * @netdev: network interface device structure
3061  *
3062  * Returns 0, this is not allowed to fail
3063  *
3064  * The close entry point is called when an interface is de-activated
3065  * by the OS.  The hardware is still under the drivers control, but
3066  * needs to be disabled.  A global MAC reset is issued to stop the
3067  * hardware, and all transmit and receive resources are freed.
3068  **/
3069 static int ixgbevf_close(struct net_device *netdev)
3070 {
3071         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3072
3073         ixgbevf_down(adapter);
3074         ixgbevf_free_irq(adapter);
3075
3076         ixgbevf_free_all_tx_resources(adapter);
3077         ixgbevf_free_all_rx_resources(adapter);
3078
3079         return 0;
3080 }
3081
3082 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
3083 {
3084         struct net_device *dev = adapter->netdev;
3085
3086         if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
3087                 return;
3088
3089         adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
3090
3091         /* if interface is down do nothing */
3092         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3093             test_bit(__IXGBEVF_RESETTING, &adapter->state))
3094                 return;
3095
3096         /* Hardware has to reinitialize queues and interrupts to
3097          * match packet buffer alignment. Unfortunately, the
3098          * hardware is not flexible enough to do this dynamically.
3099          */
3100         if (netif_running(dev))
3101                 ixgbevf_close(dev);
3102
3103         ixgbevf_clear_interrupt_scheme(adapter);
3104         ixgbevf_init_interrupt_scheme(adapter);
3105
3106         if (netif_running(dev))
3107                 ixgbevf_open(dev);
3108 }
3109
3110 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
3111                                 u32 vlan_macip_lens, u32 type_tucmd,
3112                                 u32 mss_l4len_idx)
3113 {
3114         struct ixgbe_adv_tx_context_desc *context_desc;
3115         u16 i = tx_ring->next_to_use;
3116
3117         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
3118
3119         i++;
3120         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3121
3122         /* set bits to identify this as an advanced context descriptor */
3123         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3124
3125         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
3126         context_desc->seqnum_seed       = 0;
3127         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
3128         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
3129 }
3130
3131 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
3132                        struct ixgbevf_tx_buffer *first,
3133                        u8 *hdr_len)
3134 {
3135         struct sk_buff *skb = first->skb;
3136         u32 vlan_macip_lens, type_tucmd;
3137         u32 mss_l4len_idx, l4len;
3138         int err;
3139
3140         if (skb->ip_summed != CHECKSUM_PARTIAL)
3141                 return 0;
3142
3143         if (!skb_is_gso(skb))
3144                 return 0;
3145
3146         err = skb_cow_head(skb, 0);
3147         if (err < 0)
3148                 return err;
3149
3150         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3151         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3152
3153         if (first->protocol == htons(ETH_P_IP)) {
3154                 struct iphdr *iph = ip_hdr(skb);
3155                 iph->tot_len = 0;
3156                 iph->check = 0;
3157                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3158                                                          iph->daddr, 0,
3159                                                          IPPROTO_TCP,
3160                                                          0);
3161                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3162                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3163                                    IXGBE_TX_FLAGS_CSUM |
3164                                    IXGBE_TX_FLAGS_IPV4;
3165         } else if (skb_is_gso_v6(skb)) {
3166                 ipv6_hdr(skb)->payload_len = 0;
3167                 tcp_hdr(skb)->check =
3168                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3169                                      &ipv6_hdr(skb)->daddr,
3170                                      0, IPPROTO_TCP, 0);
3171                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3172                                    IXGBE_TX_FLAGS_CSUM;
3173         }
3174
3175         /* compute header lengths */
3176         l4len = tcp_hdrlen(skb);
3177         *hdr_len += l4len;
3178         *hdr_len = skb_transport_offset(skb) + l4len;
3179
3180         /* update gso size and bytecount with header size */
3181         first->gso_segs = skb_shinfo(skb)->gso_segs;
3182         first->bytecount += (first->gso_segs - 1) * *hdr_len;
3183
3184         /* mss_l4len_id: use 1 as index for TSO */
3185         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
3186         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
3187         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
3188
3189         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
3190         vlan_macip_lens = skb_network_header_len(skb);
3191         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3192         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3193
3194         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3195                             type_tucmd, mss_l4len_idx);
3196
3197         return 1;
3198 }
3199
3200 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
3201                             struct ixgbevf_tx_buffer *first)
3202 {
3203         struct sk_buff *skb = first->skb;
3204         u32 vlan_macip_lens = 0;
3205         u32 mss_l4len_idx = 0;
3206         u32 type_tucmd = 0;
3207
3208         if (skb->ip_summed == CHECKSUM_PARTIAL) {
3209                 u8 l4_hdr = 0;
3210                 switch (first->protocol) {
3211                 case htons(ETH_P_IP):
3212                         vlan_macip_lens |= skb_network_header_len(skb);
3213                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3214                         l4_hdr = ip_hdr(skb)->protocol;
3215                         break;
3216                 case htons(ETH_P_IPV6):
3217                         vlan_macip_lens |= skb_network_header_len(skb);
3218                         l4_hdr = ipv6_hdr(skb)->nexthdr;
3219                         break;
3220                 default:
3221                         if (unlikely(net_ratelimit())) {
3222                                 dev_warn(tx_ring->dev,
3223                                  "partial checksum but proto=%x!\n",
3224                                  first->protocol);
3225                         }
3226                         break;
3227                 }
3228
3229                 switch (l4_hdr) {
3230                 case IPPROTO_TCP:
3231                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3232                         mss_l4len_idx = tcp_hdrlen(skb) <<
3233                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3234                         break;
3235                 case IPPROTO_SCTP:
3236                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3237                         mss_l4len_idx = sizeof(struct sctphdr) <<
3238                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3239                         break;
3240                 case IPPROTO_UDP:
3241                         mss_l4len_idx = sizeof(struct udphdr) <<
3242                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3243                         break;
3244                 default:
3245                         if (unlikely(net_ratelimit())) {
3246                                 dev_warn(tx_ring->dev,
3247                                  "partial checksum but l4 proto=%x!\n",
3248                                  l4_hdr);
3249                         }
3250                         break;
3251                 }
3252
3253                 /* update TX checksum flag */
3254                 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
3255         }
3256
3257         /* vlan_macip_lens: MACLEN, VLAN tag */
3258         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3259         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3260
3261         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3262                             type_tucmd, mss_l4len_idx);
3263 }
3264
3265 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
3266 {
3267         /* set type for advanced descriptor with frame checksum insertion */
3268         __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
3269                                       IXGBE_ADVTXD_DCMD_IFCS |
3270                                       IXGBE_ADVTXD_DCMD_DEXT);
3271
3272         /* set HW vlan bit if vlan is present */
3273         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3274                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
3275
3276         /* set segmentation enable bits for TSO/FSO */
3277         if (tx_flags & IXGBE_TX_FLAGS_TSO)
3278                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
3279
3280         return cmd_type;
3281 }
3282
3283 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
3284                                      u32 tx_flags, unsigned int paylen)
3285 {
3286         __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
3287
3288         /* enable L4 checksum for TSO and TX checksum offload */
3289         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3290                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
3291
3292         /* enble IPv4 checksum for TSO */
3293         if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3294                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
3295
3296         /* use index 1 context for TSO/FSO/FCOE */
3297         if (tx_flags & IXGBE_TX_FLAGS_TSO)
3298                 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
3299
3300         /* Check Context must be set if Tx switch is enabled, which it
3301          * always is for case where virtual functions are running
3302          */
3303         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
3304
3305         tx_desc->read.olinfo_status = olinfo_status;
3306 }
3307
3308 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3309                            struct ixgbevf_tx_buffer *first,
3310                            const u8 hdr_len)
3311 {
3312         dma_addr_t dma;
3313         struct sk_buff *skb = first->skb;
3314         struct ixgbevf_tx_buffer *tx_buffer;
3315         union ixgbe_adv_tx_desc *tx_desc;
3316         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
3317         unsigned int data_len = skb->data_len;
3318         unsigned int size = skb_headlen(skb);
3319         unsigned int paylen = skb->len - hdr_len;
3320         u32 tx_flags = first->tx_flags;
3321         __le32 cmd_type;
3322         u16 i = tx_ring->next_to_use;
3323
3324         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
3325
3326         ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
3327         cmd_type = ixgbevf_tx_cmd_type(tx_flags);
3328
3329         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3330         if (dma_mapping_error(tx_ring->dev, dma))
3331                 goto dma_error;
3332
3333         /* record length, and DMA address */
3334         dma_unmap_len_set(first, len, size);
3335         dma_unmap_addr_set(first, dma, dma);
3336
3337         tx_desc->read.buffer_addr = cpu_to_le64(dma);
3338
3339         for (;;) {
3340                 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
3341                         tx_desc->read.cmd_type_len =
3342                                 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
3343
3344                         i++;
3345                         tx_desc++;
3346                         if (i == tx_ring->count) {
3347                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3348                                 i = 0;
3349                         }
3350
3351                         dma += IXGBE_MAX_DATA_PER_TXD;
3352                         size -= IXGBE_MAX_DATA_PER_TXD;
3353
3354                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
3355                         tx_desc->read.olinfo_status = 0;
3356                 }
3357
3358                 if (likely(!data_len))
3359                         break;
3360
3361                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
3362
3363                 i++;
3364                 tx_desc++;
3365                 if (i == tx_ring->count) {
3366                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3367                         i = 0;
3368                 }
3369
3370                 size = skb_frag_size(frag);
3371                 data_len -= size;
3372
3373                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3374                                        DMA_TO_DEVICE);
3375                 if (dma_mapping_error(tx_ring->dev, dma))
3376                         goto dma_error;
3377
3378                 tx_buffer = &tx_ring->tx_buffer_info[i];
3379                 dma_unmap_len_set(tx_buffer, len, size);
3380                 dma_unmap_addr_set(tx_buffer, dma, dma);
3381
3382                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3383                 tx_desc->read.olinfo_status = 0;
3384
3385                 frag++;
3386         }
3387
3388         /* write last descriptor with RS and EOP bits */
3389         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3390         tx_desc->read.cmd_type_len = cmd_type;
3391
3392         /* set the timestamp */
3393         first->time_stamp = jiffies;
3394
3395         /* Force memory writes to complete before letting h/w know there
3396          * are new descriptors to fetch.  (Only applicable for weak-ordered
3397          * memory model archs, such as IA-64).
3398          *
3399          * We also need this memory barrier (wmb) to make certain all of the
3400          * status bits have been updated before next_to_watch is written.
3401          */
3402         wmb();
3403
3404         /* set next_to_watch value indicating a packet is present */
3405         first->next_to_watch = tx_desc;
3406
3407         i++;
3408         if (i == tx_ring->count)
3409                 i = 0;
3410
3411         tx_ring->next_to_use = i;
3412
3413         /* notify HW of packet */
3414         ixgbevf_write_tail(tx_ring, i);
3415
3416         return;
3417 dma_error:
3418         dev_err(tx_ring->dev, "TX DMA map failed\n");
3419
3420         /* clear dma mappings for failed tx_buffer_info map */
3421         for (;;) {
3422                 tx_buffer = &tx_ring->tx_buffer_info[i];
3423                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3424                 if (tx_buffer == first)
3425                         break;
3426                 if (i == 0)
3427                         i = tx_ring->count;
3428                 i--;
3429         }
3430
3431         tx_ring->next_to_use = i;
3432 }
3433
3434 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3435 {
3436         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
3437         /* Herbert's original patch had:
3438          *  smp_mb__after_netif_stop_queue();
3439          * but since that doesn't exist yet, just open code it. */
3440         smp_mb();
3441
3442         /* We need to check again in a case another CPU has just
3443          * made room available. */
3444         if (likely(ixgbevf_desc_unused(tx_ring) < size))
3445                 return -EBUSY;
3446
3447         /* A reprieve! - use start_queue because it doesn't call schedule */
3448         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
3449         ++tx_ring->tx_stats.restart_queue;
3450
3451         return 0;
3452 }
3453
3454 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3455 {
3456         if (likely(ixgbevf_desc_unused(tx_ring) >= size))
3457                 return 0;
3458         return __ixgbevf_maybe_stop_tx(tx_ring, size);
3459 }
3460
3461 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3462 {
3463         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3464         struct ixgbevf_tx_buffer *first;
3465         struct ixgbevf_ring *tx_ring;
3466         int tso;
3467         u32 tx_flags = 0;
3468         u16 count = TXD_USE_COUNT(skb_headlen(skb));
3469 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3470         unsigned short f;
3471 #endif
3472         u8 hdr_len = 0;
3473         u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
3474
3475         if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
3476                 dev_kfree_skb(skb);
3477                 return NETDEV_TX_OK;
3478         }
3479
3480         tx_ring = adapter->tx_ring[skb->queue_mapping];
3481
3482         /*
3483          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3484          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3485          *       + 2 desc gap to keep tail from touching head,
3486          *       + 1 desc for context descriptor,
3487          * otherwise try next time
3488          */
3489 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3490         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3491                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3492 #else
3493         count += skb_shinfo(skb)->nr_frags;
3494 #endif
3495         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3496                 tx_ring->tx_stats.tx_busy++;
3497                 return NETDEV_TX_BUSY;
3498         }
3499
3500         /* record the location of the first descriptor for this packet */
3501         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3502         first->skb = skb;
3503         first->bytecount = skb->len;
3504         first->gso_segs = 1;
3505
3506         if (skb_vlan_tag_present(skb)) {
3507                 tx_flags |= skb_vlan_tag_get(skb);
3508                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3509                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3510         }
3511
3512         /* record initial flags and protocol */
3513         first->tx_flags = tx_flags;
3514         first->protocol = vlan_get_protocol(skb);
3515
3516         tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3517         if (tso < 0)
3518                 goto out_drop;
3519         else if (!tso)
3520                 ixgbevf_tx_csum(tx_ring, first);
3521
3522         ixgbevf_tx_map(tx_ring, first, hdr_len);
3523
3524         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
3525
3526         return NETDEV_TX_OK;
3527
3528 out_drop:
3529         dev_kfree_skb_any(first->skb);
3530         first->skb = NULL;
3531
3532         return NETDEV_TX_OK;
3533 }
3534
3535 /**
3536  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3537  * @netdev: network interface device structure
3538  * @p: pointer to an address structure
3539  *
3540  * Returns 0 on success, negative on failure
3541  **/
3542 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3543 {
3544         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3545         struct ixgbe_hw *hw = &adapter->hw;
3546         struct sockaddr *addr = p;
3547
3548         if (!is_valid_ether_addr(addr->sa_data))
3549                 return -EADDRNOTAVAIL;
3550
3551         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3552         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3553
3554         spin_lock_bh(&adapter->mbx_lock);
3555
3556         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3557
3558         spin_unlock_bh(&adapter->mbx_lock);
3559
3560         return 0;
3561 }
3562
3563 /**
3564  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3565  * @netdev: network interface device structure
3566  * @new_mtu: new value for maximum frame size
3567  *
3568  * Returns 0 on success, negative on failure
3569  **/
3570 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3571 {
3572         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3573         struct ixgbe_hw *hw = &adapter->hw;
3574         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3575         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3576
3577         switch (adapter->hw.api_version) {
3578         case ixgbe_mbox_api_11:
3579                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3580                 break;
3581         default:
3582                 if (adapter->hw.mac.type != ixgbe_mac_82599_vf)
3583                         max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3584                 break;
3585         }
3586
3587         /* MTU < 68 is an error and causes problems on some kernels */
3588         if ((new_mtu < 68) || (max_frame > max_possible_frame))
3589                 return -EINVAL;
3590
3591         hw_dbg(hw, "changing MTU from %d to %d\n",
3592                netdev->mtu, new_mtu);
3593         /* must set new MTU before calling down or up */
3594         netdev->mtu = new_mtu;
3595
3596         /* notify the PF of our intent to use this size of frame */
3597         ixgbevf_rlpml_set_vf(hw, max_frame);
3598
3599         return 0;
3600 }
3601
3602 #ifdef CONFIG_NET_POLL_CONTROLLER
3603 /* Polling 'interrupt' - used by things like netconsole to send skbs
3604  * without having to re-enable interrupts. It's not called while
3605  * the interrupt routine is executing.
3606  */
3607 static void ixgbevf_netpoll(struct net_device *netdev)
3608 {
3609         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3610         int i;
3611
3612         /* if interface is down do nothing */
3613         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
3614                 return;
3615         for (i = 0; i < adapter->num_rx_queues; i++)
3616                 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]);
3617 }
3618 #endif /* CONFIG_NET_POLL_CONTROLLER */
3619
3620 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
3621 {
3622         struct net_device *netdev = pci_get_drvdata(pdev);
3623         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3624 #ifdef CONFIG_PM
3625         int retval = 0;
3626 #endif
3627
3628         netif_device_detach(netdev);
3629
3630         if (netif_running(netdev)) {
3631                 rtnl_lock();
3632                 ixgbevf_down(adapter);
3633                 ixgbevf_free_irq(adapter);
3634                 ixgbevf_free_all_tx_resources(adapter);
3635                 ixgbevf_free_all_rx_resources(adapter);
3636                 rtnl_unlock();
3637         }
3638
3639         ixgbevf_clear_interrupt_scheme(adapter);
3640
3641 #ifdef CONFIG_PM
3642         retval = pci_save_state(pdev);
3643         if (retval)
3644                 return retval;
3645
3646 #endif
3647         if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3648                 pci_disable_device(pdev);
3649
3650         return 0;
3651 }
3652
3653 #ifdef CONFIG_PM
3654 static int ixgbevf_resume(struct pci_dev *pdev)
3655 {
3656         struct net_device *netdev = pci_get_drvdata(pdev);
3657         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3658         u32 err;
3659
3660         pci_restore_state(pdev);
3661         /*
3662          * pci_restore_state clears dev->state_saved so call
3663          * pci_save_state to restore it.
3664          */
3665         pci_save_state(pdev);
3666
3667         err = pci_enable_device_mem(pdev);
3668         if (err) {
3669                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3670                 return err;
3671         }
3672         smp_mb__before_atomic();
3673         clear_bit(__IXGBEVF_DISABLED, &adapter->state);
3674         pci_set_master(pdev);
3675
3676         ixgbevf_reset(adapter);
3677
3678         rtnl_lock();
3679         err = ixgbevf_init_interrupt_scheme(adapter);
3680         rtnl_unlock();
3681         if (err) {
3682                 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3683                 return err;
3684         }
3685
3686         if (netif_running(netdev)) {
3687                 err = ixgbevf_open(netdev);
3688                 if (err)
3689                         return err;
3690         }
3691
3692         netif_device_attach(netdev);
3693
3694         return err;
3695 }
3696
3697 #endif /* CONFIG_PM */
3698 static void ixgbevf_shutdown(struct pci_dev *pdev)
3699 {
3700         ixgbevf_suspend(pdev, PMSG_SUSPEND);
3701 }
3702
3703 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3704                                                 struct rtnl_link_stats64 *stats)
3705 {
3706         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3707         unsigned int start;
3708         u64 bytes, packets;
3709         const struct ixgbevf_ring *ring;
3710         int i;
3711
3712         ixgbevf_update_stats(adapter);
3713
3714         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3715
3716         for (i = 0; i < adapter->num_rx_queues; i++) {
3717                 ring = adapter->rx_ring[i];
3718                 do {
3719                         start = u64_stats_fetch_begin_irq(&ring->syncp);
3720                         bytes = ring->stats.bytes;
3721                         packets = ring->stats.packets;
3722                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
3723                 stats->rx_bytes += bytes;
3724                 stats->rx_packets += packets;
3725         }
3726
3727         for (i = 0; i < adapter->num_tx_queues; i++) {
3728                 ring = adapter->tx_ring[i];
3729                 do {
3730                         start = u64_stats_fetch_begin_irq(&ring->syncp);
3731                         bytes = ring->stats.bytes;
3732                         packets = ring->stats.packets;
3733                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
3734                 stats->tx_bytes += bytes;
3735                 stats->tx_packets += packets;
3736         }
3737
3738         return stats;
3739 }
3740
3741 static const struct net_device_ops ixgbevf_netdev_ops = {
3742         .ndo_open               = ixgbevf_open,
3743         .ndo_stop               = ixgbevf_close,
3744         .ndo_start_xmit         = ixgbevf_xmit_frame,
3745         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3746         .ndo_get_stats64        = ixgbevf_get_stats,
3747         .ndo_validate_addr      = eth_validate_addr,
3748         .ndo_set_mac_address    = ixgbevf_set_mac,
3749         .ndo_change_mtu         = ixgbevf_change_mtu,
3750         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3751         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3752         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3753 #ifdef CONFIG_NET_RX_BUSY_POLL
3754         .ndo_busy_poll          = ixgbevf_busy_poll_recv,
3755 #endif
3756 #ifdef CONFIG_NET_POLL_CONTROLLER
3757         .ndo_poll_controller    = ixgbevf_netpoll,
3758 #endif
3759 };
3760
3761 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3762 {
3763         dev->netdev_ops = &ixgbevf_netdev_ops;
3764         ixgbevf_set_ethtool_ops(dev);
3765         dev->watchdog_timeo = 5 * HZ;
3766 }
3767
3768 /**
3769  * ixgbevf_probe - Device Initialization Routine
3770  * @pdev: PCI device information struct
3771  * @ent: entry in ixgbevf_pci_tbl
3772  *
3773  * Returns 0 on success, negative on failure
3774  *
3775  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3776  * The OS initialization, configuring of the adapter private structure,
3777  * and a hardware reset occur.
3778  **/
3779 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3780 {
3781         struct net_device *netdev;
3782         struct ixgbevf_adapter *adapter = NULL;
3783         struct ixgbe_hw *hw = NULL;
3784         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3785         int err, pci_using_dac;
3786         bool disable_dev = false;
3787
3788         err = pci_enable_device(pdev);
3789         if (err)
3790                 return err;
3791
3792         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
3793                 pci_using_dac = 1;
3794         } else {
3795                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3796                 if (err) {
3797                         dev_err(&pdev->dev, "No usable DMA "
3798                                 "configuration, aborting\n");
3799                         goto err_dma;
3800                 }
3801                 pci_using_dac = 0;
3802         }
3803
3804         err = pci_request_regions(pdev, ixgbevf_driver_name);
3805         if (err) {
3806                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3807                 goto err_pci_reg;
3808         }
3809
3810         pci_set_master(pdev);
3811
3812         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3813                                    MAX_TX_QUEUES);
3814         if (!netdev) {
3815                 err = -ENOMEM;
3816                 goto err_alloc_etherdev;
3817         }
3818
3819         SET_NETDEV_DEV(netdev, &pdev->dev);
3820
3821         adapter = netdev_priv(netdev);
3822
3823         adapter->netdev = netdev;
3824         adapter->pdev = pdev;
3825         hw = &adapter->hw;
3826         hw->back = adapter;
3827         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3828
3829         /*
3830          * call save state here in standalone driver because it relies on
3831          * adapter struct to exist, and needs to call netdev_priv
3832          */
3833         pci_save_state(pdev);
3834
3835         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3836                               pci_resource_len(pdev, 0));
3837         adapter->io_addr = hw->hw_addr;
3838         if (!hw->hw_addr) {
3839                 err = -EIO;
3840                 goto err_ioremap;
3841         }
3842
3843         ixgbevf_assign_netdev_ops(netdev);
3844
3845         /* Setup hw api */
3846         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3847         hw->mac.type  = ii->mac;
3848
3849         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3850                sizeof(struct ixgbe_mbx_operations));
3851
3852         /* setup the private structure */
3853         err = ixgbevf_sw_init(adapter);
3854         if (err)
3855                 goto err_sw_init;
3856
3857         /* The HW MAC address was set and/or determined in sw_init */
3858         if (!is_valid_ether_addr(netdev->dev_addr)) {
3859                 pr_err("invalid MAC address\n");
3860                 err = -EIO;
3861                 goto err_sw_init;
3862         }
3863
3864         netdev->hw_features = NETIF_F_SG |
3865                            NETIF_F_IP_CSUM |
3866                            NETIF_F_IPV6_CSUM |
3867                            NETIF_F_TSO |
3868                            NETIF_F_TSO6 |
3869                            NETIF_F_RXCSUM;
3870
3871         netdev->features = netdev->hw_features |
3872                            NETIF_F_HW_VLAN_CTAG_TX |
3873                            NETIF_F_HW_VLAN_CTAG_RX |
3874                            NETIF_F_HW_VLAN_CTAG_FILTER;
3875
3876         netdev->vlan_features |= NETIF_F_TSO;
3877         netdev->vlan_features |= NETIF_F_TSO6;
3878         netdev->vlan_features |= NETIF_F_IP_CSUM;
3879         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3880         netdev->vlan_features |= NETIF_F_SG;
3881
3882         if (pci_using_dac)
3883                 netdev->features |= NETIF_F_HIGHDMA;
3884
3885         netdev->priv_flags |= IFF_UNICAST_FLT;
3886
3887         init_timer(&adapter->watchdog_timer);
3888         adapter->watchdog_timer.function = ixgbevf_watchdog;
3889         adapter->watchdog_timer.data = (unsigned long)adapter;
3890
3891         if (IXGBE_REMOVED(hw->hw_addr)) {
3892                 err = -EIO;
3893                 goto err_sw_init;
3894         }
3895         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3896         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3897         set_bit(__IXGBEVF_WORK_INIT, &adapter->state);
3898
3899         err = ixgbevf_init_interrupt_scheme(adapter);
3900         if (err)
3901                 goto err_sw_init;
3902
3903         strcpy(netdev->name, "eth%d");
3904
3905         err = register_netdev(netdev);
3906         if (err)
3907                 goto err_register;
3908
3909         pci_set_drvdata(pdev, netdev);
3910         netif_carrier_off(netdev);
3911
3912         ixgbevf_init_last_counter_stats(adapter);
3913
3914         /* print the VF info */
3915         dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
3916         dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type);
3917
3918         switch (hw->mac.type) {
3919         case ixgbe_mac_X550_vf:
3920                 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n");
3921                 break;
3922         case ixgbe_mac_X540_vf:
3923                 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n");
3924                 break;
3925         case ixgbe_mac_82599_vf:
3926         default:
3927                 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n");
3928                 break;
3929         }
3930
3931         return 0;
3932
3933 err_register:
3934         ixgbevf_clear_interrupt_scheme(adapter);
3935 err_sw_init:
3936         ixgbevf_reset_interrupt_capability(adapter);
3937         iounmap(adapter->io_addr);
3938 err_ioremap:
3939         disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
3940         free_netdev(netdev);
3941 err_alloc_etherdev:
3942         pci_release_regions(pdev);
3943 err_pci_reg:
3944 err_dma:
3945         if (!adapter || disable_dev)
3946                 pci_disable_device(pdev);
3947         return err;
3948 }
3949
3950 /**
3951  * ixgbevf_remove - Device Removal Routine
3952  * @pdev: PCI device information struct
3953  *
3954  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3955  * that it should release a PCI device.  The could be caused by a
3956  * Hot-Plug event, or because the driver is going to be removed from
3957  * memory.
3958  **/
3959 static void ixgbevf_remove(struct pci_dev *pdev)
3960 {
3961         struct net_device *netdev = pci_get_drvdata(pdev);
3962         struct ixgbevf_adapter *adapter;
3963         bool disable_dev;
3964
3965         if (!netdev)
3966                 return;
3967
3968         adapter = netdev_priv(netdev);
3969
3970         set_bit(__IXGBEVF_REMOVING, &adapter->state);
3971
3972         del_timer_sync(&adapter->watchdog_timer);
3973
3974         cancel_work_sync(&adapter->reset_task);
3975         cancel_work_sync(&adapter->watchdog_task);
3976
3977         if (netdev->reg_state == NETREG_REGISTERED)
3978                 unregister_netdev(netdev);
3979
3980         ixgbevf_clear_interrupt_scheme(adapter);
3981         ixgbevf_reset_interrupt_capability(adapter);
3982
3983         iounmap(adapter->io_addr);
3984         pci_release_regions(pdev);
3985
3986         hw_dbg(&adapter->hw, "Remove complete\n");
3987
3988         disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
3989         free_netdev(netdev);
3990
3991         if (disable_dev)
3992                 pci_disable_device(pdev);
3993 }
3994
3995 /**
3996  * ixgbevf_io_error_detected - called when PCI error is detected
3997  * @pdev: Pointer to PCI device
3998  * @state: The current pci connection state
3999  *
4000  * This function is called after a PCI bus error affecting
4001  * this device has been detected.
4002  */
4003 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
4004                                                   pci_channel_state_t state)
4005 {
4006         struct net_device *netdev = pci_get_drvdata(pdev);
4007         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4008
4009         if (!test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
4010                 return PCI_ERS_RESULT_DISCONNECT;
4011
4012         rtnl_lock();
4013         netif_device_detach(netdev);
4014
4015         if (state == pci_channel_io_perm_failure) {
4016                 rtnl_unlock();
4017                 return PCI_ERS_RESULT_DISCONNECT;
4018         }
4019
4020         if (netif_running(netdev))
4021                 ixgbevf_down(adapter);
4022
4023         if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
4024                 pci_disable_device(pdev);
4025         rtnl_unlock();
4026
4027         /* Request a slot slot reset. */
4028         return PCI_ERS_RESULT_NEED_RESET;
4029 }
4030
4031 /**
4032  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
4033  * @pdev: Pointer to PCI device
4034  *
4035  * Restart the card from scratch, as if from a cold-boot. Implementation
4036  * resembles the first-half of the ixgbevf_resume routine.
4037  */
4038 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
4039 {
4040         struct net_device *netdev = pci_get_drvdata(pdev);
4041         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4042
4043         if (pci_enable_device_mem(pdev)) {
4044                 dev_err(&pdev->dev,
4045                         "Cannot re-enable PCI device after reset.\n");
4046                 return PCI_ERS_RESULT_DISCONNECT;
4047         }
4048
4049         smp_mb__before_atomic();
4050         clear_bit(__IXGBEVF_DISABLED, &adapter->state);
4051         pci_set_master(pdev);
4052
4053         ixgbevf_reset(adapter);
4054
4055         return PCI_ERS_RESULT_RECOVERED;
4056 }
4057
4058 /**
4059  * ixgbevf_io_resume - called when traffic can start flowing again.
4060  * @pdev: Pointer to PCI device
4061  *
4062  * This callback is called when the error recovery driver tells us that
4063  * its OK to resume normal operation. Implementation resembles the
4064  * second-half of the ixgbevf_resume routine.
4065  */
4066 static void ixgbevf_io_resume(struct pci_dev *pdev)
4067 {
4068         struct net_device *netdev = pci_get_drvdata(pdev);
4069         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4070
4071         if (netif_running(netdev))
4072                 ixgbevf_up(adapter);
4073
4074         netif_device_attach(netdev);
4075 }
4076
4077 /* PCI Error Recovery (ERS) */
4078 static const struct pci_error_handlers ixgbevf_err_handler = {
4079         .error_detected = ixgbevf_io_error_detected,
4080         .slot_reset = ixgbevf_io_slot_reset,
4081         .resume = ixgbevf_io_resume,
4082 };
4083
4084 static struct pci_driver ixgbevf_driver = {
4085         .name     = ixgbevf_driver_name,
4086         .id_table = ixgbevf_pci_tbl,
4087         .probe    = ixgbevf_probe,
4088         .remove   = ixgbevf_remove,
4089 #ifdef CONFIG_PM
4090         /* Power Management Hooks */
4091         .suspend  = ixgbevf_suspend,
4092         .resume   = ixgbevf_resume,
4093 #endif
4094         .shutdown = ixgbevf_shutdown,
4095         .err_handler = &ixgbevf_err_handler
4096 };
4097
4098 /**
4099  * ixgbevf_init_module - Driver Registration Routine
4100  *
4101  * ixgbevf_init_module is the first routine called when the driver is
4102  * loaded. All it does is register with the PCI subsystem.
4103  **/
4104 static int __init ixgbevf_init_module(void)
4105 {
4106         int ret;
4107         pr_info("%s - version %s\n", ixgbevf_driver_string,
4108                 ixgbevf_driver_version);
4109
4110         pr_info("%s\n", ixgbevf_copyright);
4111
4112         ret = pci_register_driver(&ixgbevf_driver);
4113         return ret;
4114 }
4115
4116 module_init(ixgbevf_init_module);
4117
4118 /**
4119  * ixgbevf_exit_module - Driver Exit Cleanup Routine
4120  *
4121  * ixgbevf_exit_module is called just before the driver is removed
4122  * from memory.
4123  **/
4124 static void __exit ixgbevf_exit_module(void)
4125 {
4126         pci_unregister_driver(&ixgbevf_driver);
4127 }
4128
4129 #ifdef DEBUG
4130 /**
4131  * ixgbevf_get_hw_dev_name - return device name string
4132  * used by hardware layer to print debugging information
4133  **/
4134 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
4135 {
4136         struct ixgbevf_adapter *adapter = hw->back;
4137         return adapter->netdev->name;
4138 }
4139
4140 #endif
4141 module_exit(ixgbevf_exit_module);
4142
4143 /* ixgbevf_main.c */