drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.12.1-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69 };
  70
  71 /* ixgbevf_pci_tbl - PCI Device ID Table
  72  *
  73  * Wildcard entries (PCI_ANY_ID) should come last
  74  * Last entry must be all 0s
  75  *
  76  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  77  *   Class, Class Mask, private data (not used) }
  78  */
  79 static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
  80         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
  81         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
  82         /* required last entry */
  83         {0, }
  84 };
  85 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  86
  87 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  88 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
  89 MODULE_LICENSE("GPL");
  90 MODULE_VERSION(DRV_VERSION);
  91
  92 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  93 static int debug = -1;
  94 module_param(debug, int, 0);
  95 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  96
  97 /* forward decls */
  98 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
  99 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
 100 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
 101
 102 static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring,
 103                                            u32 val)
 104 {
 105         rx_ring->next_to_use = val;
 106
 107         /*
 108          * Force memory writes to complete before letting h/w
 109          * know there are new descriptors to fetch.  (Only
 110          * applicable for weak-ordered memory model archs,
 111          * such as IA-64).
 112          */
 113         wmb();
 114         writel(val, rx_ring->tail);
 115 }
 116
 117 /**
 118  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 119  * @adapter: pointer to adapter struct
 120  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 121  * @queue: queue to map the corresponding interrupt to
 122  * @msix_vector: the vector to map to the corresponding queue
 123  */
 124 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 125                              u8 queue, u8 msix_vector)
 126 {
 127         u32 ivar, index;
 128         struct ixgbe_hw *hw = &adapter->hw;
 129         if (direction == -1) {
 130                 /* other causes */
 131                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 132                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 133                 ivar &= ~0xFF;
 134                 ivar |= msix_vector;
 135                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 136         } else {
 137                 /* tx or rx causes */
 138                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 139                 index = ((16 * (queue & 1)) + (8 * direction));
 140                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 141                 ivar &= ~(0xFF << index);
 142                 ivar |= (msix_vector << index);
 143                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 144         }
 145 }
 146
 147 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 148                                         struct ixgbevf_tx_buffer *tx_buffer)
 149 {
 150         if (tx_buffer->skb) {
 151                 dev_kfree_skb_any(tx_buffer->skb);
 152                 if (dma_unmap_len(tx_buffer, len))
 153                         dma_unmap_single(tx_ring->dev,
 154                                          dma_unmap_addr(tx_buffer, dma),
 155                                          dma_unmap_len(tx_buffer, len),
 156                                          DMA_TO_DEVICE);
 157         } else if (dma_unmap_len(tx_buffer, len)) {
 158                 dma_unmap_page(tx_ring->dev,
 159                                dma_unmap_addr(tx_buffer, dma),
 160                                dma_unmap_len(tx_buffer, len),
 161                                DMA_TO_DEVICE);
 162         }
 163         tx_buffer->next_to_watch = NULL;
 164         tx_buffer->skb = NULL;
 165         dma_unmap_len_set(tx_buffer, len, 0);
 166         /* tx_buffer must be completely set up in the transmit path */
 167 }
 168
 169 #define IXGBE_MAX_TXD_PWR       14
 170 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 171
 172 /* Tx Descriptors needed, worst case */
 173 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 174 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 175
 176 static void ixgbevf_tx_timeout(struct net_device *netdev);
 177
 178 /**
 179  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 180  * @q_vector: board private structure
 181  * @tx_ring: tx ring to clean
 182  **/
 183 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 184                                  struct ixgbevf_ring *tx_ring)
 185 {
 186         struct ixgbevf_adapter *adapter = q_vector->adapter;
 187         struct ixgbevf_tx_buffer *tx_buffer;
 188         union ixgbe_adv_tx_desc *tx_desc;
 189         unsigned int total_bytes = 0, total_packets = 0;
 190         unsigned int budget = tx_ring->count / 2;
 191         unsigned int i = tx_ring->next_to_clean;
 192
 193         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 194                 return true;
 195
 196         tx_buffer = &tx_ring->tx_buffer_info[i];
 197         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 198         i -= tx_ring->count;
 199
 200         do {
 201                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
 202
 203                 /* if next_to_watch is not set then there is no work pending */
 204                 if (!eop_desc)
 205                         break;
 206
 207                 /* prevent any other reads prior to eop_desc */
 208                 read_barrier_depends();
 209
 210                 /* if DD is not set pending work has not been completed */
 211                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 212                         break;
 213
 214                 /* clear next_to_watch to prevent false hangs */
 215                 tx_buffer->next_to_watch = NULL;
 216
 217                 /* update the statistics for this packet */
 218                 total_bytes += tx_buffer->bytecount;
 219                 total_packets += tx_buffer->gso_segs;
 220
 221                 /* free the skb */
 222                 dev_kfree_skb_any(tx_buffer->skb);
 223
 224                 /* unmap skb header data */
 225                 dma_unmap_single(tx_ring->dev,
 226                                  dma_unmap_addr(tx_buffer, dma),
 227                                  dma_unmap_len(tx_buffer, len),
 228                                  DMA_TO_DEVICE);
 229
 230                 /* clear tx_buffer data */
 231                 tx_buffer->skb = NULL;
 232                 dma_unmap_len_set(tx_buffer, len, 0);
 233
 234                 /* unmap remaining buffers */
 235                 while (tx_desc != eop_desc) {
 236                         tx_buffer++;
 237                         tx_desc++;
 238                         i++;
 239                         if (unlikely(!i)) {
 240                                 i -= tx_ring->count;
 241                                 tx_buffer = tx_ring->tx_buffer_info;
 242                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 243                         }
 244
 245                         /* unmap any remaining paged data */
 246                         if (dma_unmap_len(tx_buffer, len)) {
 247                                 dma_unmap_page(tx_ring->dev,
 248                                                dma_unmap_addr(tx_buffer, dma),
 249                                                dma_unmap_len(tx_buffer, len),
 250                                                DMA_TO_DEVICE);
 251                                 dma_unmap_len_set(tx_buffer, len, 0);
 252                         }
 253                 }
 254
 255                 /* move us one more past the eop_desc for start of next pkt */
 256                 tx_buffer++;
 257                 tx_desc++;
 258                 i++;
 259                 if (unlikely(!i)) {
 260                         i -= tx_ring->count;
 261                         tx_buffer = tx_ring->tx_buffer_info;
 262                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 263                 }
 264
 265                 /* issue prefetch for next Tx descriptor */
 266                 prefetch(tx_desc);
 267
 268                 /* update budget accounting */
 269                 budget--;
 270         } while (likely(budget));
 271
 272         i += tx_ring->count;
 273         tx_ring->next_to_clean = i;
 274         u64_stats_update_begin(&tx_ring->syncp);
 275         tx_ring->stats.bytes += total_bytes;
 276         tx_ring->stats.packets += total_packets;
 277         u64_stats_update_end(&tx_ring->syncp);
 278         q_vector->tx.total_bytes += total_bytes;
 279         q_vector->tx.total_packets += total_packets;
 280
 281 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 282         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 283                      (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 284                 /* Make sure that anybody stopping the queue after this
 285                  * sees the new next_to_clean.
 286                  */
 287                 smp_mb();
 288
 289                 if (__netif_subqueue_stopped(tx_ring->netdev,
 290                                              tx_ring->queue_index) &&
 291                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 292                         netif_wake_subqueue(tx_ring->netdev,
 293                                             tx_ring->queue_index);
 294                         ++tx_ring->tx_stats.restart_queue;
 295                 }
 296         }
 297
 298         return !!budget;
 299 }
 300
 301 /**
 302  * ixgbevf_receive_skb - Send a completed packet up the stack
 303  * @q_vector: structure containing interrupt and ring information
 304  * @skb: packet to send up
 305  * @status: hardware indication of status of receive
 306  * @rx_desc: rx descriptor
 307  **/
 308 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
 309                                 struct sk_buff *skb, u8 status,
 310                                 union ixgbe_adv_rx_desc *rx_desc)
 311 {
 312         struct ixgbevf_adapter *adapter = q_vector->adapter;
 313         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
 314         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
 315
 316         if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
 317                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
 318
 319         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
 320                 napi_gro_receive(&q_vector->napi, skb);
 321         else
 322                 netif_rx(skb);
 323 }
 324
 325 /**
 326  * ixgbevf_rx_skb - Helper function to determine proper Rx method
 327  * @q_vector: structure containing interrupt and ring information
 328  * @skb: packet to send up
 329  * @status: hardware indication of status of receive
 330  * @rx_desc: rx descriptor
 331  **/
 332 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
 333                            struct sk_buff *skb, u8 status,
 334                            union ixgbe_adv_rx_desc *rx_desc)
 335 {
 336 #ifdef CONFIG_NET_RX_BUSY_POLL
 337         skb_mark_napi_id(skb, &q_vector->napi);
 338
 339         if (ixgbevf_qv_busy_polling(q_vector)) {
 340                 netif_receive_skb(skb);
 341                 /* exit early if we busy polled */
 342                 return;
 343         }
 344 #endif /* CONFIG_NET_RX_BUSY_POLL */
 345
 346         ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
 347 }
 348
 349 /**
 350  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 351  * @ring: pointer to Rx descriptor ring structure
 352  * @status_err: hardware indication of status of receive
 353  * @skb: skb currently being received and modified
 354  **/
 355 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
 356                                        u32 status_err, struct sk_buff *skb)
 357 {
 358         skb_checksum_none_assert(skb);
 359
 360         /* Rx csum disabled */
 361         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 362                 return;
 363
 364         /* if IP and error */
 365         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
 366             (status_err & IXGBE_RXDADV_ERR_IPE)) {
 367                 ring->rx_stats.csum_err++;
 368                 return;
 369         }
 370
 371         if (!(status_err & IXGBE_RXD_STAT_L4CS))
 372                 return;
 373
 374         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
 375                 ring->rx_stats.csum_err++;
 376                 return;
 377         }
 378
 379         /* It must be a TCP or UDP packet with a valid checksum */
 380         skb->ip_summed = CHECKSUM_UNNECESSARY;
 381 }
 382
 383 /**
 384  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 385  * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
 386  **/
 387 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
 388                                      int cleaned_count)
 389 {
 390         union ixgbe_adv_rx_desc *rx_desc;
 391         struct ixgbevf_rx_buffer *bi;
 392         unsigned int i = rx_ring->next_to_use;
 393
 394         while (cleaned_count--) {
 395                 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 396                 bi = &rx_ring->rx_buffer_info[i];
 397
 398                 if (!bi->skb) {
 399                         struct sk_buff *skb;
 400
 401                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 402                                                         rx_ring->rx_buf_len);
 403                         if (!skb)
 404                                 goto no_buffers;
 405
 406                         bi->skb = skb;
 407
 408                         bi->dma = dma_map_single(rx_ring->dev, skb->data,
 409                                                  rx_ring->rx_buf_len,
 410                                                  DMA_FROM_DEVICE);
 411                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
 412                                 dev_kfree_skb(skb);
 413                                 bi->skb = NULL;
 414                                 dev_err(rx_ring->dev, "Rx DMA map failed\n");
 415                                 break;
 416                         }
 417                 }
 418                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
 419
 420                 i++;
 421                 if (i == rx_ring->count)
 422                         i = 0;
 423         }
 424
 425 no_buffers:
 426         rx_ring->rx_stats.alloc_rx_buff_failed++;
 427         if (rx_ring->next_to_use != i)
 428                 ixgbevf_release_rx_desc(rx_ring, i);
 429 }
 430
 431 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 432                                              u32 qmask)
 433 {
 434         struct ixgbe_hw *hw = &adapter->hw;
 435
 436         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 437 }
 438
 439 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 440                                 struct ixgbevf_ring *rx_ring,
 441                                 int budget)
 442 {
 443         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
 444         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
 445         struct sk_buff *skb;
 446         unsigned int i;
 447         u32 len, staterr;
 448         int cleaned_count = 0;
 449         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 450
 451         i = rx_ring->next_to_clean;
 452         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 453         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 454         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 455
 456         while (staterr & IXGBE_RXD_STAT_DD) {
 457                 if (!budget)
 458                         break;
 459                 budget--;
 460
 461                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 462                 len = le16_to_cpu(rx_desc->wb.upper.length);
 463                 skb = rx_buffer_info->skb;
 464                 prefetch(skb->data - NET_IP_ALIGN);
 465                 rx_buffer_info->skb = NULL;
 466
 467                 if (rx_buffer_info->dma) {
 468                         dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
 469                                          rx_ring->rx_buf_len,
 470                                          DMA_FROM_DEVICE);
 471                         rx_buffer_info->dma = 0;
 472                         skb_put(skb, len);
 473                 }
 474
 475                 i++;
 476                 if (i == rx_ring->count)
 477                         i = 0;
 478
 479                 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
 480                 prefetch(next_rxd);
 481                 cleaned_count++;
 482
 483                 next_buffer = &rx_ring->rx_buffer_info[i];
 484
 485                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
 486                         skb->next = next_buffer->skb;
 487                         IXGBE_CB(skb->next)->prev = skb;
 488                         rx_ring->rx_stats.non_eop_descs++;
 489                         goto next_desc;
 490                 }
 491
 492                 /* we should not be chaining buffers, if we did drop the skb */
 493                 if (IXGBE_CB(skb)->prev) {
 494                         do {
 495                                 struct sk_buff *this = skb;
 496                                 skb = IXGBE_CB(skb)->prev;
 497                                 dev_kfree_skb(this);
 498                         } while (skb);
 499                         goto next_desc;
 500                 }
 501
 502                 /* ERR_MASK will only have valid bits if EOP set */
 503                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
 504                         dev_kfree_skb_irq(skb);
 505                         goto next_desc;
 506                 }
 507
 508                 ixgbevf_rx_checksum(rx_ring, staterr, skb);
 509
 510                 /* probably a little skewed due to removing CRC */
 511                 total_rx_bytes += skb->len;
 512                 total_rx_packets++;
 513
 514                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 515
 516                 /* Workaround hardware that can't do proper VEPA multicast
 517                  * source pruning.
 518                  */
 519                 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
 520                     ether_addr_equal(rx_ring->netdev->dev_addr,
 521                                      eth_hdr(skb)->h_source)) {
 522                         dev_kfree_skb_irq(skb);
 523                         goto next_desc;
 524                 }
 525
 526                 ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
 527
 528 next_desc:
 529                 rx_desc->wb.upper.status_error = 0;
 530
 531                 /* return some buffers to hardware, one at a time is too slow */
 532                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 533                         ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
 534                         cleaned_count = 0;
 535                 }
 536
 537                 /* use prefetched values */
 538                 rx_desc = next_rxd;
 539                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
 540
 541                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 542         }
 543
 544         rx_ring->next_to_clean = i;
 545         cleaned_count = ixgbevf_desc_unused(rx_ring);
 546
 547         if (cleaned_count)
 548                 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
 549
 550         u64_stats_update_begin(&rx_ring->syncp);
 551         rx_ring->stats.packets += total_rx_packets;
 552         rx_ring->stats.bytes += total_rx_bytes;
 553         u64_stats_update_end(&rx_ring->syncp);
 554         q_vector->rx.total_packets += total_rx_packets;
 555         q_vector->rx.total_bytes += total_rx_bytes;
 556
 557         return total_rx_packets;
 558 }
 559
 560 /**
 561  * ixgbevf_poll - NAPI polling calback
 562  * @napi: napi struct with our devices info in it
 563  * @budget: amount of work driver is allowed to do this pass, in packets
 564  *
 565  * This function will clean more than one or more rings associated with a
 566  * q_vector.
 567  **/
 568 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 569 {
 570         struct ixgbevf_q_vector *q_vector =
 571                 container_of(napi, struct ixgbevf_q_vector, napi);
 572         struct ixgbevf_adapter *adapter = q_vector->adapter;
 573         struct ixgbevf_ring *ring;
 574         int per_ring_budget;
 575         bool clean_complete = true;
 576
 577         ixgbevf_for_each_ring(ring, q_vector->tx)
 578                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 579
 580 #ifdef CONFIG_NET_RX_BUSY_POLL
 581         if (!ixgbevf_qv_lock_napi(q_vector))
 582                 return budget;
 583 #endif
 584
 585         /* attempt to distribute budget to each queue fairly, but don't allow
 586          * the budget to go below 1 because we'll exit polling */
 587         if (q_vector->rx.count > 1)
 588                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 589         else
 590                 per_ring_budget = budget;
 591
 592         adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
 593         ixgbevf_for_each_ring(ring, q_vector->rx)
 594                 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
 595                                                         per_ring_budget)
 596                                    < per_ring_budget);
 597         adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
 598
 599 #ifdef CONFIG_NET_RX_BUSY_POLL
 600         ixgbevf_qv_unlock_napi(q_vector);
 601 #endif
 602
 603         /* If all work not completed, return budget and keep polling */
 604         if (!clean_complete)
 605                 return budget;
 606         /* all work done, exit the polling mode */
 607         napi_complete(napi);
 608         if (adapter->rx_itr_setting & 1)
 609                 ixgbevf_set_itr(q_vector);
 610         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 611                 ixgbevf_irq_enable_queues(adapter,
 612                                           1 << q_vector->v_idx);
 613
 614         return 0;
 615 }
 616
 617 /**
 618  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 619  * @q_vector: structure containing interrupt and ring information
 620  */
 621 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 622 {
 623         struct ixgbevf_adapter *adapter = q_vector->adapter;
 624         struct ixgbe_hw *hw = &adapter->hw;
 625         int v_idx = q_vector->v_idx;
 626         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 627
 628         /*
 629          * set the WDIS bit to not clear the timer bits and cause an
 630          * immediate assertion of the interrupt
 631          */
 632         itr_reg |= IXGBE_EITR_CNT_WDIS;
 633
 634         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 635 }
 636
 637 #ifdef CONFIG_NET_RX_BUSY_POLL
 638 /* must be called with local_bh_disable()d */
 639 static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
 640 {
 641         struct ixgbevf_q_vector *q_vector =
 642                         container_of(napi, struct ixgbevf_q_vector, napi);
 643         struct ixgbevf_adapter *adapter = q_vector->adapter;
 644         struct ixgbevf_ring  *ring;
 645         int found = 0;
 646
 647         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 648                 return LL_FLUSH_FAILED;
 649
 650         if (!ixgbevf_qv_lock_poll(q_vector))
 651                 return LL_FLUSH_BUSY;
 652
 653         ixgbevf_for_each_ring(ring, q_vector->rx) {
 654                 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
 655 #ifdef BP_EXTENDED_STATS
 656                 if (found)
 657                         ring->stats.cleaned += found;
 658                 else
 659                         ring->stats.misses++;
 660 #endif
 661                 if (found)
 662                         break;
 663         }
 664
 665         ixgbevf_qv_unlock_poll(q_vector);
 666
 667         return found;
 668 }
 669 #endif /* CONFIG_NET_RX_BUSY_POLL */
 670
 671 /**
 672  * ixgbevf_configure_msix - Configure MSI-X hardware
 673  * @adapter: board private structure
 674  *
 675  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 676  * interrupts.
 677  **/
 678 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
 679 {
 680         struct ixgbevf_q_vector *q_vector;
 681         int q_vectors, v_idx;
 682
 683         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 684         adapter->eims_enable_mask = 0;
 685
 686         /*
 687          * Populate the IVAR table and set the ITR values to the
 688          * corresponding register.
 689          */
 690         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
 691                 struct ixgbevf_ring *ring;
 692                 q_vector = adapter->q_vector[v_idx];
 693
 694                 ixgbevf_for_each_ring(ring, q_vector->rx)
 695                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
 696
 697                 ixgbevf_for_each_ring(ring, q_vector->tx)
 698                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
 699
 700                 if (q_vector->tx.ring && !q_vector->rx.ring) {
 701                         /* tx only vector */
 702                         if (adapter->tx_itr_setting == 1)
 703                                 q_vector->itr = IXGBE_10K_ITR;
 704                         else
 705                                 q_vector->itr = adapter->tx_itr_setting;
 706                 } else {
 707                         /* rx or rx/tx vector */
 708                         if (adapter->rx_itr_setting == 1)
 709                                 q_vector->itr = IXGBE_20K_ITR;
 710                         else
 711                                 q_vector->itr = adapter->rx_itr_setting;
 712                 }
 713
 714                 /* add q_vector eims value to global eims_enable_mask */
 715                 adapter->eims_enable_mask |= 1 << v_idx;
 716
 717                 ixgbevf_write_eitr(q_vector);
 718         }
 719
 720         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
 721         /* setup eims_other and add value to global eims_enable_mask */
 722         adapter->eims_other = 1 << v_idx;
 723         adapter->eims_enable_mask |= adapter->eims_other;
 724 }
 725
 726 enum latency_range {
 727         lowest_latency = 0,
 728         low_latency = 1,
 729         bulk_latency = 2,
 730         latency_invalid = 255
 731 };
 732
 733 /**
 734  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
 735  * @q_vector: structure containing interrupt and ring information
 736  * @ring_container: structure containing ring performance data
 737  *
 738  *      Stores a new ITR value based on packets and byte
 739  *      counts during the last interrupt.  The advantage of per interrupt
 740  *      computation is faster updates and more accurate ITR for the current
 741  *      traffic pattern.  Constants in this function were computed
 742  *      based on theoretical maximum wire speed and thresholds were set based
 743  *      on testing data as well as attempting to minimize response time
 744  *      while increasing bulk throughput.
 745  **/
 746 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
 747                                struct ixgbevf_ring_container *ring_container)
 748 {
 749         int bytes = ring_container->total_bytes;
 750         int packets = ring_container->total_packets;
 751         u32 timepassed_us;
 752         u64 bytes_perint;
 753         u8 itr_setting = ring_container->itr;
 754
 755         if (packets == 0)
 756                 return;
 757
 758         /* simple throttlerate management
 759          *    0-20MB/s lowest (100000 ints/s)
 760          *   20-100MB/s low   (20000 ints/s)
 761          *  100-1249MB/s bulk (8000 ints/s)
 762          */
 763         /* what was last interrupt timeslice? */
 764         timepassed_us = q_vector->itr >> 2;
 765         bytes_perint = bytes / timepassed_us; /* bytes/usec */
 766
 767         switch (itr_setting) {
 768         case lowest_latency:
 769                 if (bytes_perint > 10)
 770                         itr_setting = low_latency;
 771                 break;
 772         case low_latency:
 773                 if (bytes_perint > 20)
 774                         itr_setting = bulk_latency;
 775                 else if (bytes_perint <= 10)
 776                         itr_setting = lowest_latency;
 777                 break;
 778         case bulk_latency:
 779                 if (bytes_perint <= 20)
 780                         itr_setting = low_latency;
 781                 break;
 782         }
 783
 784         /* clear work counters since we have the values we need */
 785         ring_container->total_bytes = 0;
 786         ring_container->total_packets = 0;
 787
 788         /* write updated itr to ring container */
 789         ring_container->itr = itr_setting;
 790 }
 791
 792 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
 793 {
 794         u32 new_itr = q_vector->itr;
 795         u8 current_itr;
 796
 797         ixgbevf_update_itr(q_vector, &q_vector->tx);
 798         ixgbevf_update_itr(q_vector, &q_vector->rx);
 799
 800         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
 801
 802         switch (current_itr) {
 803         /* counts and packets in update_itr are dependent on these numbers */
 804         case lowest_latency:
 805                 new_itr = IXGBE_100K_ITR;
 806                 break;
 807         case low_latency:
 808                 new_itr = IXGBE_20K_ITR;
 809                 break;
 810         case bulk_latency:
 811         default:
 812                 new_itr = IXGBE_8K_ITR;
 813                 break;
 814         }
 815
 816         if (new_itr != q_vector->itr) {
 817                 /* do an exponential smoothing */
 818                 new_itr = (10 * new_itr * q_vector->itr) /
 819                           ((9 * new_itr) + q_vector->itr);
 820
 821                 /* save the algorithm value here */
 822                 q_vector->itr = new_itr;
 823
 824                 ixgbevf_write_eitr(q_vector);
 825         }
 826 }
 827
 828 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
 829 {
 830         struct ixgbevf_adapter *adapter = data;
 831         struct ixgbe_hw *hw = &adapter->hw;
 832
 833         hw->mac.get_link_status = 1;
 834
 835         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 836                 mod_timer(&adapter->watchdog_timer, jiffies);
 837
 838         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
 839
 840         return IRQ_HANDLED;
 841 }
 842
 843 /**
 844  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
 845  * @irq: unused
 846  * @data: pointer to our q_vector struct for this interrupt vector
 847  **/
 848 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
 849 {
 850         struct ixgbevf_q_vector *q_vector = data;
 851
 852         /* EIAM disabled interrupts (on this vector) for us */
 853         if (q_vector->rx.ring || q_vector->tx.ring)
 854                 napi_schedule(&q_vector->napi);
 855
 856         return IRQ_HANDLED;
 857 }
 858
 859 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
 860                                      int r_idx)
 861 {
 862         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 863
 864         a->rx_ring[r_idx]->next = q_vector->rx.ring;
 865         q_vector->rx.ring = a->rx_ring[r_idx];
 866         q_vector->rx.count++;
 867 }
 868
 869 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
 870                                      int t_idx)
 871 {
 872         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 873
 874         a->tx_ring[t_idx]->next = q_vector->tx.ring;
 875         q_vector->tx.ring = a->tx_ring[t_idx];
 876         q_vector->tx.count++;
 877 }
 878
 879 /**
 880  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
 881  * @adapter: board private structure to initialize
 882  *
 883  * This function maps descriptor rings to the queue-specific vectors
 884  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 885  * one vector per ring/queue, but on a constrained vector budget, we
 886  * group the rings as "efficiently" as possible.  You would add new
 887  * mapping configurations in here.
 888  **/
 889 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
 890 {
 891         int q_vectors;
 892         int v_start = 0;
 893         int rxr_idx = 0, txr_idx = 0;
 894         int rxr_remaining = adapter->num_rx_queues;
 895         int txr_remaining = adapter->num_tx_queues;
 896         int i, j;
 897         int rqpv, tqpv;
 898         int err = 0;
 899
 900         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 901
 902         /*
 903          * The ideal configuration...
 904          * We have enough vectors to map one per queue.
 905          */
 906         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
 907                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 908                         map_vector_to_rxq(adapter, v_start, rxr_idx);
 909
 910                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 911                         map_vector_to_txq(adapter, v_start, txr_idx);
 912                 goto out;
 913         }
 914
 915         /*
 916          * If we don't have enough vectors for a 1-to-1
 917          * mapping, we'll have to group them so there are
 918          * multiple queues per vector.
 919          */
 920         /* Re-adjusting *qpv takes care of the remainder. */
 921         for (i = v_start; i < q_vectors; i++) {
 922                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 923                 for (j = 0; j < rqpv; j++) {
 924                         map_vector_to_rxq(adapter, i, rxr_idx);
 925                         rxr_idx++;
 926                         rxr_remaining--;
 927                 }
 928         }
 929         for (i = v_start; i < q_vectors; i++) {
 930                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 931                 for (j = 0; j < tqpv; j++) {
 932                         map_vector_to_txq(adapter, i, txr_idx);
 933                         txr_idx++;
 934                         txr_remaining--;
 935                 }
 936         }
 937
 938 out:
 939         return err;
 940 }
 941
 942 /**
 943  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
 944  * @adapter: board private structure
 945  *
 946  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
 947  * interrupts from the kernel.
 948  **/
 949 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
 950 {
 951         struct net_device *netdev = adapter->netdev;
 952         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 953         int vector, err;
 954         int ri = 0, ti = 0;
 955
 956         for (vector = 0; vector < q_vectors; vector++) {
 957                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
 958                 struct msix_entry *entry = &adapter->msix_entries[vector];
 959
 960                 if (q_vector->tx.ring && q_vector->rx.ring) {
 961                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 962                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
 963                         ti++;
 964                 } else if (q_vector->rx.ring) {
 965                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 966                                  "%s-%s-%d", netdev->name, "rx", ri++);
 967                 } else if (q_vector->tx.ring) {
 968                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 969                                  "%s-%s-%d", netdev->name, "tx", ti++);
 970                 } else {
 971                         /* skip this unused q_vector */
 972                         continue;
 973                 }
 974                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
 975                                   q_vector->name, q_vector);
 976                 if (err) {
 977                         hw_dbg(&adapter->hw,
 978                                "request_irq failed for MSIX interrupt "
 979                                "Error: %d\n", err);
 980                         goto free_queue_irqs;
 981                 }
 982         }
 983
 984         err = request_irq(adapter->msix_entries[vector].vector,
 985                           &ixgbevf_msix_other, 0, netdev->name, adapter);
 986         if (err) {
 987                 hw_dbg(&adapter->hw,
 988                        "request_irq for msix_other failed: %d\n", err);
 989                 goto free_queue_irqs;
 990         }
 991
 992         return 0;
 993
 994 free_queue_irqs:
 995         while (vector) {
 996                 vector--;
 997                 free_irq(adapter->msix_entries[vector].vector,
 998                          adapter->q_vector[vector]);
 999         }
1000         /* This failure is non-recoverable - it indicates the system is
1001          * out of MSIX vector resources and the VF driver cannot run
1002          * without them.  Set the number of msix vectors to zero
1003          * indicating that not enough can be allocated.  The error
1004          * will be returned to the user indicating device open failed.
1005          * Any further attempts to force the driver to open will also
1006          * fail.  The only way to recover is to unload the driver and
1007          * reload it again.  If the system has recovered some MSIX
1008          * vectors then it may succeed.
1009          */
1010         adapter->num_msix_vectors = 0;
1011         return err;
1012 }
1013
1014 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1015 {
1016         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1017
1018         for (i = 0; i < q_vectors; i++) {
1019                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1020                 q_vector->rx.ring = NULL;
1021                 q_vector->tx.ring = NULL;
1022                 q_vector->rx.count = 0;
1023                 q_vector->tx.count = 0;
1024         }
1025 }
1026
1027 /**
1028  * ixgbevf_request_irq - initialize interrupts
1029  * @adapter: board private structure
1030  *
1031  * Attempts to configure interrupts using the best available
1032  * capabilities of the hardware and kernel.
1033  **/
1034 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1035 {
1036         int err = 0;
1037
1038         err = ixgbevf_request_msix_irqs(adapter);
1039
1040         if (err)
1041                 hw_dbg(&adapter->hw,
1042                        "request_irq failed, Error %d\n", err);
1043
1044         return err;
1045 }
1046
1047 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1048 {
1049         int i, q_vectors;
1050
1051         q_vectors = adapter->num_msix_vectors;
1052         i = q_vectors - 1;
1053
1054         free_irq(adapter->msix_entries[i].vector, adapter);
1055         i--;
1056
1057         for (; i >= 0; i--) {
1058                 /* free only the irqs that were actually requested */
1059                 if (!adapter->q_vector[i]->rx.ring &&
1060                     !adapter->q_vector[i]->tx.ring)
1061                         continue;
1062
1063                 free_irq(adapter->msix_entries[i].vector,
1064                          adapter->q_vector[i]);
1065         }
1066
1067         ixgbevf_reset_q_vectors(adapter);
1068 }
1069
1070 /**
1071  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1072  * @adapter: board private structure
1073  **/
1074 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1075 {
1076         struct ixgbe_hw *hw = &adapter->hw;
1077         int i;
1078
1079         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1080         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1081         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1082
1083         IXGBE_WRITE_FLUSH(hw);
1084
1085         for (i = 0; i < adapter->num_msix_vectors; i++)
1086                 synchronize_irq(adapter->msix_entries[i].vector);
1087 }
1088
1089 /**
1090  * ixgbevf_irq_enable - Enable default interrupt generation settings
1091  * @adapter: board private structure
1092  **/
1093 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1094 {
1095         struct ixgbe_hw *hw = &adapter->hw;
1096
1097         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1098         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1099         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1100 }
1101
1102 /**
1103  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1104  * @adapter: board private structure
1105  * @ring: structure containing ring specific data
1106  *
1107  * Configure the Tx descriptor ring after a reset.
1108  **/
1109 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1110                                       struct ixgbevf_ring *ring)
1111 {
1112         struct ixgbe_hw *hw = &adapter->hw;
1113         u64 tdba = ring->dma;
1114         int wait_loop = 10;
1115         u32 txdctl = IXGBE_TXDCTL_ENABLE;
1116         u8 reg_idx = ring->reg_idx;
1117
1118         /* disable queue to avoid issues while updating state */
1119         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1120         IXGBE_WRITE_FLUSH(hw);
1121
1122         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1123         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1124         IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1125                         ring->count * sizeof(union ixgbe_adv_tx_desc));
1126
1127         /* disable head writeback */
1128         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1129         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1130
1131         /* enable relaxed ordering */
1132         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1133                         (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1134                          IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1135
1136         /* reset head and tail pointers */
1137         IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1138         IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1139         ring->tail = hw->hw_addr + IXGBE_VFTDT(reg_idx);
1140
1141         /* reset ntu and ntc to place SW in sync with hardwdare */
1142         ring->next_to_clean = 0;
1143         ring->next_to_use = 0;
1144
1145         /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1146          * to or less than the number of on chip descriptors, which is
1147          * currently 40.
1148          */
1149         txdctl |= (8 << 16);    /* WTHRESH = 8 */
1150
1151         /* Setting PTHRESH to 32 both improves performance */
1152         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
1153                   32;          /* PTHRESH = 32 */
1154
1155         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1156
1157         /* poll to verify queue is enabled */
1158         do {
1159                 usleep_range(1000, 2000);
1160                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1161         }  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1162         if (!wait_loop)
1163                 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1164 }
1165
1166 /**
1167  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1168  * @adapter: board private structure
1169  *
1170  * Configure the Tx unit of the MAC after a reset.
1171  **/
1172 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1173 {
1174         u32 i;
1175
1176         /* Setup the HW Tx Head and Tail descriptor pointers */
1177         for (i = 0; i < adapter->num_tx_queues; i++)
1178                 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
1179 }
1180
1181 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1182
1183 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1184 {
1185         struct ixgbevf_ring *rx_ring;
1186         struct ixgbe_hw *hw = &adapter->hw;
1187         u32 srrctl;
1188
1189         rx_ring = adapter->rx_ring[index];
1190
1191         srrctl = IXGBE_SRRCTL_DROP_EN;
1192
1193         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1194
1195         srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1196                   IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1197
1198         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1199 }
1200
1201 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1202 {
1203         struct ixgbe_hw *hw = &adapter->hw;
1204
1205         /* PSRTYPE must be initialized in 82599 */
1206         u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1207                       IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1208                       IXGBE_PSRTYPE_L2HDR;
1209
1210         if (adapter->num_rx_queues > 1)
1211                 psrtype |= 1 << 29;
1212
1213         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1214 }
1215
1216 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1217 {
1218         struct ixgbe_hw *hw = &adapter->hw;
1219         struct net_device *netdev = adapter->netdev;
1220         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1221         int i;
1222         u16 rx_buf_len;
1223
1224         /* notify the PF of our intent to use this size of frame */
1225         ixgbevf_rlpml_set_vf(hw, max_frame);
1226
1227         /* PF will allow an extra 4 bytes past for vlan tagged frames */
1228         max_frame += VLAN_HLEN;
1229
1230         /*
1231          * Allocate buffer sizes that fit well into 32K and
1232          * take into account max frame size of 9.5K
1233          */
1234         if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1235             (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1236                 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1237         else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1238                 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1239         else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1240                 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1241         else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1242                 rx_buf_len = IXGBEVF_RXBUFFER_8K;
1243         else
1244                 rx_buf_len = IXGBEVF_RXBUFFER_10K;
1245
1246         for (i = 0; i < adapter->num_rx_queues; i++)
1247                 adapter->rx_ring[i]->rx_buf_len = rx_buf_len;
1248 }
1249
1250 #define IXGBEVF_MAX_RX_DESC_POLL 10
1251 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1252                                      struct ixgbevf_ring *ring)
1253 {
1254         struct ixgbe_hw *hw = &adapter->hw;
1255         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1256         u32 rxdctl;
1257         u8 reg_idx = ring->reg_idx;
1258
1259         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1260         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1261
1262         /* write value back with RXDCTL.ENABLE bit cleared */
1263         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1264
1265         /* the hardware may take up to 100us to really disable the rx queue */
1266         do {
1267                 udelay(10);
1268                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1269         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1270
1271         if (!wait_loop)
1272                 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1273                        reg_idx);
1274 }
1275
1276 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1277                                          struct ixgbevf_ring *ring)
1278 {
1279         struct ixgbe_hw *hw = &adapter->hw;
1280         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1281         u32 rxdctl;
1282         u8 reg_idx = ring->reg_idx;
1283
1284         do {
1285                 usleep_range(1000, 2000);
1286                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1287         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1288
1289         if (!wait_loop)
1290                 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1291                        reg_idx);
1292 }
1293
1294 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1295                                       struct ixgbevf_ring *ring)
1296 {
1297         struct ixgbe_hw *hw = &adapter->hw;
1298         u64 rdba = ring->dma;
1299         u32 rxdctl;
1300         u8 reg_idx = ring->reg_idx;
1301
1302         /* disable queue to avoid issues while updating state */
1303         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1304         ixgbevf_disable_rx_queue(adapter, ring);
1305
1306         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1307         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1308         IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1309                         ring->count * sizeof(union ixgbe_adv_rx_desc));
1310
1311         /* enable relaxed ordering */
1312         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1313                         IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1314
1315         /* reset head and tail pointers */
1316         IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1317         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1318         ring->tail = hw->hw_addr + IXGBE_VFRDT(reg_idx);
1319
1320         /* reset ntu and ntc to place SW in sync with hardwdare */
1321         ring->next_to_clean = 0;
1322         ring->next_to_use = 0;
1323
1324         ixgbevf_configure_srrctl(adapter, reg_idx);
1325
1326         /* prevent DMA from exceeding buffer space available */
1327         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1328         rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN;
1329         rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1330         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1331
1332         ixgbevf_rx_desc_queue_enable(adapter, ring);
1333         ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
1334 }
1335
1336 /**
1337  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1338  * @adapter: board private structure
1339  *
1340  * Configure the Rx unit of the MAC after a reset.
1341  **/
1342 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1343 {
1344         int i;
1345
1346         ixgbevf_setup_psrtype(adapter);
1347
1348         /* set_rx_buffer_len must be called before ring initialization */
1349         ixgbevf_set_rx_buffer_len(adapter);
1350
1351         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1352          * the Base and Length of the Rx Descriptor Ring */
1353         for (i = 0; i < adapter->num_rx_queues; i++)
1354                 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
1355 }
1356
1357 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1358                                    __be16 proto, u16 vid)
1359 {
1360         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1361         struct ixgbe_hw *hw = &adapter->hw;
1362         int err;
1363
1364         spin_lock_bh(&adapter->mbx_lock);
1365
1366         /* add VID to filter table */
1367         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1368
1369         spin_unlock_bh(&adapter->mbx_lock);
1370
1371         /* translate error return types so error makes sense */
1372         if (err == IXGBE_ERR_MBX)
1373                 return -EIO;
1374
1375         if (err == IXGBE_ERR_INVALID_ARGUMENT)
1376                 return -EACCES;
1377
1378         set_bit(vid, adapter->active_vlans);
1379
1380         return err;
1381 }
1382
1383 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1384                                     __be16 proto, u16 vid)
1385 {
1386         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1387         struct ixgbe_hw *hw = &adapter->hw;
1388         int err = -EOPNOTSUPP;
1389
1390         spin_lock_bh(&adapter->mbx_lock);
1391
1392         /* remove VID from filter table */
1393         err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1394
1395         spin_unlock_bh(&adapter->mbx_lock);
1396
1397         clear_bit(vid, adapter->active_vlans);
1398
1399         return err;
1400 }
1401
1402 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1403 {
1404         u16 vid;
1405
1406         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1407                 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1408                                         htons(ETH_P_8021Q), vid);
1409 }
1410
1411 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1412 {
1413         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1414         struct ixgbe_hw *hw = &adapter->hw;
1415         int count = 0;
1416
1417         if ((netdev_uc_count(netdev)) > 10) {
1418                 pr_err("Too many unicast filters - No Space\n");
1419                 return -ENOSPC;
1420         }
1421
1422         if (!netdev_uc_empty(netdev)) {
1423                 struct netdev_hw_addr *ha;
1424                 netdev_for_each_uc_addr(ha, netdev) {
1425                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1426                         udelay(200);
1427                 }
1428         } else {
1429                 /*
1430                  * If the list is empty then send message to PF driver to
1431                  * clear all macvlans on this VF.
1432                  */
1433                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1434         }
1435
1436         return count;
1437 }
1438
1439 /**
1440  * ixgbevf_set_rx_mode - Multicast and unicast set
1441  * @netdev: network interface device structure
1442  *
1443  * The set_rx_method entry point is called whenever the multicast address
1444  * list, unicast address list or the network interface flags are updated.
1445  * This routine is responsible for configuring the hardware for proper
1446  * multicast mode and configuring requested unicast filters.
1447  **/
1448 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1449 {
1450         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1451         struct ixgbe_hw *hw = &adapter->hw;
1452
1453         spin_lock_bh(&adapter->mbx_lock);
1454
1455         /* reprogram multicast list */
1456         hw->mac.ops.update_mc_addr_list(hw, netdev);
1457
1458         ixgbevf_write_uc_addr_list(netdev);
1459
1460         spin_unlock_bh(&adapter->mbx_lock);
1461 }
1462
1463 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1464 {
1465         int q_idx;
1466         struct ixgbevf_q_vector *q_vector;
1467         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1468
1469         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1470                 q_vector = adapter->q_vector[q_idx];
1471 #ifdef CONFIG_NET_RX_BUSY_POLL
1472                 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1473 #endif
1474                 napi_enable(&q_vector->napi);
1475         }
1476 }
1477
1478 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1479 {
1480         int q_idx;
1481         struct ixgbevf_q_vector *q_vector;
1482         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1483
1484         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1485                 q_vector = adapter->q_vector[q_idx];
1486                 napi_disable(&q_vector->napi);
1487 #ifdef CONFIG_NET_RX_BUSY_POLL
1488                 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1489                         pr_info("QV %d locked\n", q_idx);
1490                         usleep_range(1000, 20000);
1491                 }
1492 #endif /* CONFIG_NET_RX_BUSY_POLL */
1493         }
1494 }
1495
1496 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1497 {
1498         struct ixgbe_hw *hw = &adapter->hw;
1499         unsigned int def_q = 0;
1500         unsigned int num_tcs = 0;
1501         unsigned int num_rx_queues = 1;
1502         int err;
1503
1504         spin_lock_bh(&adapter->mbx_lock);
1505
1506         /* fetch queue configuration from the PF */
1507         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1508
1509         spin_unlock_bh(&adapter->mbx_lock);
1510
1511         if (err)
1512                 return err;
1513
1514         if (num_tcs > 1) {
1515                 /* update default Tx ring register index */
1516                 adapter->tx_ring[0]->reg_idx = def_q;
1517
1518                 /* we need as many queues as traffic classes */
1519                 num_rx_queues = num_tcs;
1520         }
1521
1522         /* if we have a bad config abort request queue reset */
1523         if (adapter->num_rx_queues != num_rx_queues) {
1524                 /* force mailbox timeout to prevent further messages */
1525                 hw->mbx.timeout = 0;
1526
1527                 /* wait for watchdog to come around and bail us out */
1528                 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1529         }
1530
1531         return 0;
1532 }
1533
1534 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1535 {
1536         ixgbevf_configure_dcb(adapter);
1537
1538         ixgbevf_set_rx_mode(adapter->netdev);
1539
1540         ixgbevf_restore_vlan(adapter);
1541
1542         ixgbevf_configure_tx(adapter);
1543         ixgbevf_configure_rx(adapter);
1544 }
1545
1546 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1547 {
1548         /* Only save pre-reset stats if there are some */
1549         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1550                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1551                         adapter->stats.base_vfgprc;
1552                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1553                         adapter->stats.base_vfgptc;
1554                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1555                         adapter->stats.base_vfgorc;
1556                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1557                         adapter->stats.base_vfgotc;
1558                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1559                         adapter->stats.base_vfmprc;
1560         }
1561 }
1562
1563 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1564 {
1565         struct ixgbe_hw *hw = &adapter->hw;
1566
1567         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1568         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1569         adapter->stats.last_vfgorc |=
1570                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1571         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1572         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1573         adapter->stats.last_vfgotc |=
1574                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1575         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1576
1577         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1578         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1579         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1580         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1581         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1582 }
1583
1584 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1585 {
1586         struct ixgbe_hw *hw = &adapter->hw;
1587         int api[] = { ixgbe_mbox_api_11,
1588                       ixgbe_mbox_api_10,
1589                       ixgbe_mbox_api_unknown };
1590         int err = 0, idx = 0;
1591
1592         spin_lock_bh(&adapter->mbx_lock);
1593
1594         while (api[idx] != ixgbe_mbox_api_unknown) {
1595                 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1596                 if (!err)
1597                         break;
1598                 idx++;
1599         }
1600
1601         spin_unlock_bh(&adapter->mbx_lock);
1602 }
1603
1604 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1605 {
1606         struct net_device *netdev = adapter->netdev;
1607         struct ixgbe_hw *hw = &adapter->hw;
1608
1609         ixgbevf_configure_msix(adapter);
1610
1611         spin_lock_bh(&adapter->mbx_lock);
1612
1613         if (is_valid_ether_addr(hw->mac.addr))
1614                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1615         else
1616                 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1617
1618         spin_unlock_bh(&adapter->mbx_lock);
1619
1620         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1621         ixgbevf_napi_enable_all(adapter);
1622
1623         /* enable transmits */
1624         netif_tx_start_all_queues(netdev);
1625
1626         ixgbevf_save_reset_stats(adapter);
1627         ixgbevf_init_last_counter_stats(adapter);
1628
1629         hw->mac.get_link_status = 1;
1630         mod_timer(&adapter->watchdog_timer, jiffies);
1631 }
1632
1633 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1634 {
1635         struct ixgbe_hw *hw = &adapter->hw;
1636
1637         ixgbevf_configure(adapter);
1638
1639         ixgbevf_up_complete(adapter);
1640
1641         /* clear any pending interrupts, may auto mask */
1642         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1643
1644         ixgbevf_irq_enable(adapter);
1645 }
1646
1647 /**
1648  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1649  * @rx_ring: ring to free buffers from
1650  **/
1651 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
1652 {
1653         unsigned long size;
1654         unsigned int i;
1655
1656         if (!rx_ring->rx_buffer_info)
1657                 return;
1658
1659         /* Free all the Rx ring sk_buffs */
1660         for (i = 0; i < rx_ring->count; i++) {
1661                 struct ixgbevf_rx_buffer *rx_buffer_info;
1662
1663                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1664                 if (rx_buffer_info->dma) {
1665                         dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
1666                                          rx_ring->rx_buf_len,
1667                                          DMA_FROM_DEVICE);
1668                         rx_buffer_info->dma = 0;
1669                 }
1670                 if (rx_buffer_info->skb) {
1671                         struct sk_buff *skb = rx_buffer_info->skb;
1672                         rx_buffer_info->skb = NULL;
1673                         do {
1674                                 struct sk_buff *this = skb;
1675                                 skb = IXGBE_CB(skb)->prev;
1676                                 dev_kfree_skb(this);
1677                         } while (skb);
1678                 }
1679         }
1680
1681         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1682         memset(rx_ring->rx_buffer_info, 0, size);
1683
1684         /* Zero out the descriptor ring */
1685         memset(rx_ring->desc, 0, rx_ring->size);
1686 }
1687
1688 /**
1689  * ixgbevf_clean_tx_ring - Free Tx Buffers
1690  * @tx_ring: ring to be cleaned
1691  **/
1692 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
1693 {
1694         struct ixgbevf_tx_buffer *tx_buffer_info;
1695         unsigned long size;
1696         unsigned int i;
1697
1698         if (!tx_ring->tx_buffer_info)
1699                 return;
1700
1701         /* Free all the Tx ring sk_buffs */
1702         for (i = 0; i < tx_ring->count; i++) {
1703                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1704                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1705         }
1706
1707         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1708         memset(tx_ring->tx_buffer_info, 0, size);
1709
1710         memset(tx_ring->desc, 0, tx_ring->size);
1711 }
1712
1713 /**
1714  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1715  * @adapter: board private structure
1716  **/
1717 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1718 {
1719         int i;
1720
1721         for (i = 0; i < adapter->num_rx_queues; i++)
1722                 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
1723 }
1724
1725 /**
1726  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1727  * @adapter: board private structure
1728  **/
1729 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1730 {
1731         int i;
1732
1733         for (i = 0; i < adapter->num_tx_queues; i++)
1734                 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
1735 }
1736
1737 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1738 {
1739         struct net_device *netdev = adapter->netdev;
1740         struct ixgbe_hw *hw = &adapter->hw;
1741         int i;
1742
1743         /* signal that we are down to the interrupt handler */
1744         set_bit(__IXGBEVF_DOWN, &adapter->state);
1745
1746         /* disable all enabled rx queues */
1747         for (i = 0; i < adapter->num_rx_queues; i++)
1748                 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
1749
1750         netif_tx_disable(netdev);
1751
1752         msleep(10);
1753
1754         netif_tx_stop_all_queues(netdev);
1755
1756         ixgbevf_irq_disable(adapter);
1757
1758         ixgbevf_napi_disable_all(adapter);
1759
1760         del_timer_sync(&adapter->watchdog_timer);
1761         /* can't call flush scheduled work here because it can deadlock
1762          * if linkwatch_event tries to acquire the rtnl_lock which we are
1763          * holding */
1764         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1765                 msleep(1);
1766
1767         /* disable transmits in the hardware now that interrupts are off */
1768         for (i = 0; i < adapter->num_tx_queues; i++) {
1769                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
1770
1771                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
1772                                 IXGBE_TXDCTL_SWFLSH);
1773         }
1774
1775         netif_carrier_off(netdev);
1776
1777         if (!pci_channel_offline(adapter->pdev))
1778                 ixgbevf_reset(adapter);
1779
1780         ixgbevf_clean_all_tx_rings(adapter);
1781         ixgbevf_clean_all_rx_rings(adapter);
1782 }
1783
1784 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1785 {
1786         WARN_ON(in_interrupt());
1787
1788         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1789                 msleep(1);
1790
1791         ixgbevf_down(adapter);
1792         ixgbevf_up(adapter);
1793
1794         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1795 }
1796
1797 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1798 {
1799         struct ixgbe_hw *hw = &adapter->hw;
1800         struct net_device *netdev = adapter->netdev;
1801
1802         if (hw->mac.ops.reset_hw(hw)) {
1803                 hw_dbg(hw, "PF still resetting\n");
1804         } else {
1805                 hw->mac.ops.init_hw(hw);
1806                 ixgbevf_negotiate_api(adapter);
1807         }
1808
1809         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1810                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1811                        netdev->addr_len);
1812                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1813                        netdev->addr_len);
1814         }
1815 }
1816
1817 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1818                                         int vectors)
1819 {
1820         int vector_threshold;
1821
1822         /* We'll want at least 2 (vector_threshold):
1823          * 1) TxQ[0] + RxQ[0] handler
1824          * 2) Other (Link Status Change, etc.)
1825          */
1826         vector_threshold = MIN_MSIX_COUNT;
1827
1828         /* The more we get, the more we will assign to Tx/Rx Cleanup
1829          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1830          * Right now, we simply care about how many we'll get; we'll
1831          * set them up later while requesting irq's.
1832          */
1833         vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1834                                         vector_threshold, vectors);
1835
1836         if (vectors < 0) {
1837                 dev_err(&adapter->pdev->dev,
1838                         "Unable to allocate MSI-X interrupts\n");
1839                 kfree(adapter->msix_entries);
1840                 adapter->msix_entries = NULL;
1841                 return vectors;
1842         }
1843
1844         /* Adjust for only the vectors we'll use, which is minimum
1845          * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1846          * vectors we were allocated.
1847          */
1848         adapter->num_msix_vectors = vectors;
1849
1850         return 0;
1851 }
1852
1853 /**
1854  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1855  * @adapter: board private structure to initialize
1856  *
1857  * This is the top level queue allocation routine.  The order here is very
1858  * important, starting with the "most" number of features turned on at once,
1859  * and ending with the smallest set of features.  This way large combinations
1860  * can be allocated if they're turned on, and smaller combinations are the
1861  * fallthrough conditions.
1862  *
1863  **/
1864 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1865 {
1866         struct ixgbe_hw *hw = &adapter->hw;
1867         unsigned int def_q = 0;
1868         unsigned int num_tcs = 0;
1869         int err;
1870
1871         /* Start with base case */
1872         adapter->num_rx_queues = 1;
1873         adapter->num_tx_queues = 1;
1874
1875         spin_lock_bh(&adapter->mbx_lock);
1876
1877         /* fetch queue configuration from the PF */
1878         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1879
1880         spin_unlock_bh(&adapter->mbx_lock);
1881
1882         if (err)
1883                 return;
1884
1885         /* we need as many queues as traffic classes */
1886         if (num_tcs > 1)
1887                 adapter->num_rx_queues = num_tcs;
1888 }
1889
1890 /**
1891  * ixgbevf_alloc_queues - Allocate memory for all rings
1892  * @adapter: board private structure to initialize
1893  *
1894  * We allocate one ring per queue at run-time since we don't know the
1895  * number of queues at compile-time.  The polling_netdev array is
1896  * intended for Multiqueue, but should work fine with a single queue.
1897  **/
1898 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1899 {
1900         struct ixgbevf_ring *ring;
1901         int rx = 0, tx = 0;
1902
1903         for (; tx < adapter->num_tx_queues; tx++) {
1904                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1905                 if (!ring)
1906                         goto err_allocation;
1907
1908                 ring->dev = &adapter->pdev->dev;
1909                 ring->netdev = adapter->netdev;
1910                 ring->count = adapter->tx_ring_count;
1911                 ring->queue_index = tx;
1912                 ring->reg_idx = tx;
1913
1914                 adapter->tx_ring[tx] = ring;
1915         }
1916
1917         for (; rx < adapter->num_rx_queues; rx++) {
1918                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1919                 if (!ring)
1920                         goto err_allocation;
1921
1922                 ring->dev = &adapter->pdev->dev;
1923                 ring->netdev = adapter->netdev;
1924
1925                 ring->count = adapter->rx_ring_count;
1926                 ring->queue_index = rx;
1927                 ring->reg_idx = rx;
1928
1929                 adapter->rx_ring[rx] = ring;
1930         }
1931
1932         return 0;
1933
1934 err_allocation:
1935         while (tx) {
1936                 kfree(adapter->tx_ring[--tx]);
1937                 adapter->tx_ring[tx] = NULL;
1938         }
1939
1940         while (rx) {
1941                 kfree(adapter->rx_ring[--rx]);
1942                 adapter->rx_ring[rx] = NULL;
1943         }
1944         return -ENOMEM;
1945 }
1946
1947 /**
1948  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1949  * @adapter: board private structure to initialize
1950  *
1951  * Attempt to configure the interrupts using the best available
1952  * capabilities of the hardware and the kernel.
1953  **/
1954 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1955 {
1956         struct net_device *netdev = adapter->netdev;
1957         int err = 0;
1958         int vector, v_budget;
1959
1960         /*
1961          * It's easy to be greedy for MSI-X vectors, but it really
1962          * doesn't do us much good if we have a lot more vectors
1963          * than CPU's.  So let's be conservative and only ask for
1964          * (roughly) the same number of vectors as there are CPU's.
1965          * The default is to use pairs of vectors.
1966          */
1967         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1968         v_budget = min_t(int, v_budget, num_online_cpus());
1969         v_budget += NON_Q_VECTORS;
1970
1971         /* A failure in MSI-X entry allocation isn't fatal, but it does
1972          * mean we disable MSI-X capabilities of the adapter. */
1973         adapter->msix_entries = kcalloc(v_budget,
1974                                         sizeof(struct msix_entry), GFP_KERNEL);
1975         if (!adapter->msix_entries) {
1976                 err = -ENOMEM;
1977                 goto out;
1978         }
1979
1980         for (vector = 0; vector < v_budget; vector++)
1981                 adapter->msix_entries[vector].entry = vector;
1982
1983         err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1984         if (err)
1985                 goto out;
1986
1987         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1988         if (err)
1989                 goto out;
1990
1991         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1992
1993 out:
1994         return err;
1995 }
1996
1997 /**
1998  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1999  * @adapter: board private structure to initialize
2000  *
2001  * We allocate one q_vector per queue interrupt.  If allocation fails we
2002  * return -ENOMEM.
2003  **/
2004 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2005 {
2006         int q_idx, num_q_vectors;
2007         struct ixgbevf_q_vector *q_vector;
2008
2009         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2010
2011         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2012                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2013                 if (!q_vector)
2014                         goto err_out;
2015                 q_vector->adapter = adapter;
2016                 q_vector->v_idx = q_idx;
2017                 netif_napi_add(adapter->netdev, &q_vector->napi,
2018                                ixgbevf_poll, 64);
2019 #ifdef CONFIG_NET_RX_BUSY_POLL
2020                 napi_hash_add(&q_vector->napi);
2021 #endif
2022                 adapter->q_vector[q_idx] = q_vector;
2023         }
2024
2025         return 0;
2026
2027 err_out:
2028         while (q_idx) {
2029                 q_idx--;
2030                 q_vector = adapter->q_vector[q_idx];
2031 #ifdef CONFIG_NET_RX_BUSY_POLL
2032                 napi_hash_del(&q_vector->napi);
2033 #endif
2034                 netif_napi_del(&q_vector->napi);
2035                 kfree(q_vector);
2036                 adapter->q_vector[q_idx] = NULL;
2037         }
2038         return -ENOMEM;
2039 }
2040
2041 /**
2042  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2043  * @adapter: board private structure to initialize
2044  *
2045  * This function frees the memory allocated to the q_vectors.  In addition if
2046  * NAPI is enabled it will delete any references to the NAPI struct prior
2047  * to freeing the q_vector.
2048  **/
2049 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2050 {
2051         int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2052
2053         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2054                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2055
2056                 adapter->q_vector[q_idx] = NULL;
2057 #ifdef CONFIG_NET_RX_BUSY_POLL
2058                 napi_hash_del(&q_vector->napi);
2059 #endif
2060                 netif_napi_del(&q_vector->napi);
2061                 kfree(q_vector);
2062         }
2063 }
2064
2065 /**
2066  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2067  * @adapter: board private structure
2068  *
2069  **/
2070 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2071 {
2072         pci_disable_msix(adapter->pdev);
2073         kfree(adapter->msix_entries);
2074         adapter->msix_entries = NULL;
2075 }
2076
2077 /**
2078  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2079  * @adapter: board private structure to initialize
2080  *
2081  **/
2082 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2083 {
2084         int err;
2085
2086         /* Number of supported queues */
2087         ixgbevf_set_num_queues(adapter);
2088
2089         err = ixgbevf_set_interrupt_capability(adapter);
2090         if (err) {
2091                 hw_dbg(&adapter->hw,
2092                        "Unable to setup interrupt capabilities\n");
2093                 goto err_set_interrupt;
2094         }
2095
2096         err = ixgbevf_alloc_q_vectors(adapter);
2097         if (err) {
2098                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2099                        "vectors\n");
2100                 goto err_alloc_q_vectors;
2101         }
2102
2103         err = ixgbevf_alloc_queues(adapter);
2104         if (err) {
2105                 pr_err("Unable to allocate memory for queues\n");
2106                 goto err_alloc_queues;
2107         }
2108
2109         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2110                "Tx Queue count = %u\n",
2111                (adapter->num_rx_queues > 1) ? "Enabled" :
2112                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2113
2114         set_bit(__IXGBEVF_DOWN, &adapter->state);
2115
2116         return 0;
2117 err_alloc_queues:
2118         ixgbevf_free_q_vectors(adapter);
2119 err_alloc_q_vectors:
2120         ixgbevf_reset_interrupt_capability(adapter);
2121 err_set_interrupt:
2122         return err;
2123 }
2124
2125 /**
2126  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2127  * @adapter: board private structure to clear interrupt scheme on
2128  *
2129  * We go through and clear interrupt specific resources and reset the structure
2130  * to pre-load conditions
2131  **/
2132 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2133 {
2134         int i;
2135
2136         for (i = 0; i < adapter->num_tx_queues; i++) {
2137                 kfree(adapter->tx_ring[i]);
2138                 adapter->tx_ring[i] = NULL;
2139         }
2140         for (i = 0; i < adapter->num_rx_queues; i++) {
2141                 kfree(adapter->rx_ring[i]);
2142                 adapter->rx_ring[i] = NULL;
2143         }
2144
2145         adapter->num_tx_queues = 0;
2146         adapter->num_rx_queues = 0;
2147
2148         ixgbevf_free_q_vectors(adapter);
2149         ixgbevf_reset_interrupt_capability(adapter);
2150 }
2151
2152 /**
2153  * ixgbevf_sw_init - Initialize general software structures
2154  * (struct ixgbevf_adapter)
2155  * @adapter: board private structure to initialize
2156  *
2157  * ixgbevf_sw_init initializes the Adapter private data structure.
2158  * Fields are initialized based on PCI device information and
2159  * OS network device settings (MTU size).
2160  **/
2161 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2162 {
2163         struct ixgbe_hw *hw = &adapter->hw;
2164         struct pci_dev *pdev = adapter->pdev;
2165         struct net_device *netdev = adapter->netdev;
2166         int err;
2167
2168         /* PCI config space info */
2169
2170         hw->vendor_id = pdev->vendor;
2171         hw->device_id = pdev->device;
2172         hw->revision_id = pdev->revision;
2173         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2174         hw->subsystem_device_id = pdev->subsystem_device;
2175
2176         hw->mbx.ops.init_params(hw);
2177
2178         /* assume legacy case in which PF would only give VF 2 queues */
2179         hw->mac.max_tx_queues = 2;
2180         hw->mac.max_rx_queues = 2;
2181
2182         /* lock to protect mailbox accesses */
2183         spin_lock_init(&adapter->mbx_lock);
2184
2185         err = hw->mac.ops.reset_hw(hw);
2186         if (err) {
2187                 dev_info(&pdev->dev,
2188                          "PF still in reset state.  Is the PF interface up?\n");
2189         } else {
2190                 err = hw->mac.ops.init_hw(hw);
2191                 if (err) {
2192                         pr_err("init_shared_code failed: %d\n", err);
2193                         goto out;
2194                 }
2195                 ixgbevf_negotiate_api(adapter);
2196                 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2197                 if (err)
2198                         dev_info(&pdev->dev, "Error reading MAC address\n");
2199                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2200                         dev_info(&pdev->dev,
2201                                  "MAC address not assigned by administrator.\n");
2202                 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2203         }
2204
2205         if (!is_valid_ether_addr(netdev->dev_addr)) {
2206                 dev_info(&pdev->dev, "Assigning random MAC address\n");
2207                 eth_hw_addr_random(netdev);
2208                 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
2209         }
2210
2211         /* Enable dynamic interrupt throttling rates */
2212         adapter->rx_itr_setting = 1;
2213         adapter->tx_itr_setting = 1;
2214
2215         /* set default ring sizes */
2216         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2217         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2218
2219         set_bit(__IXGBEVF_DOWN, &adapter->state);
2220         return 0;
2221
2222 out:
2223         return err;
2224 }
2225
2226 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
2227         {                                                       \
2228                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
2229                 if (current_counter < last_counter)             \
2230                         counter += 0x100000000LL;               \
2231                 last_counter = current_counter;                 \
2232                 counter &= 0xFFFFFFFF00000000LL;                \
2233                 counter |= current_counter;                     \
2234         }
2235
2236 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2237         {                                                                \
2238                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
2239                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
2240                 u64 current_counter = (current_counter_msb << 32) |      \
2241                         current_counter_lsb;                             \
2242                 if (current_counter < last_counter)                      \
2243                         counter += 0x1000000000LL;                       \
2244                 last_counter = current_counter;                          \
2245                 counter &= 0xFFFFFFF000000000LL;                         \
2246                 counter |= current_counter;                              \
2247         }
2248 /**
2249  * ixgbevf_update_stats - Update the board statistics counters.
2250  * @adapter: board private structure
2251  **/
2252 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2253 {
2254         struct ixgbe_hw *hw = &adapter->hw;
2255         int i;
2256
2257         if (!adapter->link_up)
2258                 return;
2259
2260         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2261                                 adapter->stats.vfgprc);
2262         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2263                                 adapter->stats.vfgptc);
2264         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2265                                 adapter->stats.last_vfgorc,
2266                                 adapter->stats.vfgorc);
2267         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2268                                 adapter->stats.last_vfgotc,
2269                                 adapter->stats.vfgotc);
2270         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2271                                 adapter->stats.vfmprc);
2272
2273         for (i = 0;  i  < adapter->num_rx_queues;  i++) {
2274                 adapter->hw_csum_rx_error +=
2275                         adapter->rx_ring[i]->hw_csum_rx_error;
2276                 adapter->rx_ring[i]->hw_csum_rx_error = 0;
2277         }
2278 }
2279
2280 /**
2281  * ixgbevf_watchdog - Timer Call-back
2282  * @data: pointer to adapter cast into an unsigned long
2283  **/
2284 static void ixgbevf_watchdog(unsigned long data)
2285 {
2286         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2287         struct ixgbe_hw *hw = &adapter->hw;
2288         u32 eics = 0;
2289         int i;
2290
2291         /*
2292          * Do the watchdog outside of interrupt context due to the lovely
2293          * delays that some of the newer hardware requires
2294          */
2295
2296         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2297                 goto watchdog_short_circuit;
2298
2299         /* get one bit for every active tx/rx interrupt vector */
2300         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2301                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2302                 if (qv->rx.ring || qv->tx.ring)
2303                         eics |= 1 << i;
2304         }
2305
2306         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2307
2308 watchdog_short_circuit:
2309         schedule_work(&adapter->watchdog_task);
2310 }
2311
2312 /**
2313  * ixgbevf_tx_timeout - Respond to a Tx Hang
2314  * @netdev: network interface device structure
2315  **/
2316 static void ixgbevf_tx_timeout(struct net_device *netdev)
2317 {
2318         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2319
2320         /* Do the reset outside of interrupt context */
2321         schedule_work(&adapter->reset_task);
2322 }
2323
2324 static void ixgbevf_reset_task(struct work_struct *work)
2325 {
2326         struct ixgbevf_adapter *adapter;
2327         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2328
2329         /* If we're already down or resetting, just bail */
2330         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2331             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2332                 return;
2333
2334         adapter->tx_timeout_count++;
2335
2336         ixgbevf_reinit_locked(adapter);
2337 }
2338
2339 /**
2340  * ixgbevf_watchdog_task - worker thread to bring link up
2341  * @work: pointer to work_struct containing our data
2342  **/
2343 static void ixgbevf_watchdog_task(struct work_struct *work)
2344 {
2345         struct ixgbevf_adapter *adapter = container_of(work,
2346                                                        struct ixgbevf_adapter,
2347                                                        watchdog_task);
2348         struct net_device *netdev = adapter->netdev;
2349         struct ixgbe_hw *hw = &adapter->hw;
2350         u32 link_speed = adapter->link_speed;
2351         bool link_up = adapter->link_up;
2352         s32 need_reset;
2353
2354         ixgbevf_queue_reset_subtask(adapter);
2355
2356         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2357
2358         /*
2359          * Always check the link on the watchdog because we have
2360          * no LSC interrupt
2361          */
2362         spin_lock_bh(&adapter->mbx_lock);
2363
2364         need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
2365
2366         spin_unlock_bh(&adapter->mbx_lock);
2367
2368         if (need_reset) {
2369                 adapter->link_up = link_up;
2370                 adapter->link_speed = link_speed;
2371                 netif_carrier_off(netdev);
2372                 netif_tx_stop_all_queues(netdev);
2373                 schedule_work(&adapter->reset_task);
2374                 goto pf_has_reset;
2375         }
2376         adapter->link_up = link_up;
2377         adapter->link_speed = link_speed;
2378
2379         if (link_up) {
2380                 if (!netif_carrier_ok(netdev)) {
2381                         char *link_speed_string;
2382                         switch (link_speed) {
2383                         case IXGBE_LINK_SPEED_10GB_FULL:
2384                                 link_speed_string = "10 Gbps";
2385                                 break;
2386                         case IXGBE_LINK_SPEED_1GB_FULL:
2387                                 link_speed_string = "1 Gbps";
2388                                 break;
2389                         case IXGBE_LINK_SPEED_100_FULL:
2390                                 link_speed_string = "100 Mbps";
2391                                 break;
2392                         default:
2393                                 link_speed_string = "unknown speed";
2394                                 break;
2395                         }
2396                         dev_info(&adapter->pdev->dev,
2397                                 "NIC Link is Up, %s\n", link_speed_string);
2398                         netif_carrier_on(netdev);
2399                         netif_tx_wake_all_queues(netdev);
2400                 }
2401         } else {
2402                 adapter->link_up = false;
2403                 adapter->link_speed = 0;
2404                 if (netif_carrier_ok(netdev)) {
2405                         dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
2406                         netif_carrier_off(netdev);
2407                         netif_tx_stop_all_queues(netdev);
2408                 }
2409         }
2410
2411         ixgbevf_update_stats(adapter);
2412
2413 pf_has_reset:
2414         /* Reset the timer */
2415         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2416                 mod_timer(&adapter->watchdog_timer,
2417                           round_jiffies(jiffies + (2 * HZ)));
2418
2419         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2420 }
2421
2422 /**
2423  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2424  * @tx_ring: Tx descriptor ring for a specific queue
2425  *
2426  * Free all transmit software resources
2427  **/
2428 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
2429 {
2430         ixgbevf_clean_tx_ring(tx_ring);
2431
2432         vfree(tx_ring->tx_buffer_info);
2433         tx_ring->tx_buffer_info = NULL;
2434
2435         /* if not set, then don't free */
2436         if (!tx_ring->desc)
2437                 return;
2438
2439         dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2440                           tx_ring->dma);
2441
2442         tx_ring->desc = NULL;
2443 }
2444
2445 /**
2446  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2447  * @adapter: board private structure
2448  *
2449  * Free all transmit software resources
2450  **/
2451 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2452 {
2453         int i;
2454
2455         for (i = 0; i < adapter->num_tx_queues; i++)
2456                 if (adapter->tx_ring[i]->desc)
2457                         ixgbevf_free_tx_resources(adapter->tx_ring[i]);
2458 }
2459
2460 /**
2461  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2462  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2463  *
2464  * Return 0 on success, negative on failure
2465  **/
2466 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
2467 {
2468         int size;
2469
2470         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2471         tx_ring->tx_buffer_info = vzalloc(size);
2472         if (!tx_ring->tx_buffer_info)
2473                 goto err;
2474
2475         /* round up to nearest 4K */
2476         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2477         tx_ring->size = ALIGN(tx_ring->size, 4096);
2478
2479         tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2480                                            &tx_ring->dma, GFP_KERNEL);
2481         if (!tx_ring->desc)
2482                 goto err;
2483
2484         return 0;
2485
2486 err:
2487         vfree(tx_ring->tx_buffer_info);
2488         tx_ring->tx_buffer_info = NULL;
2489         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2490                "descriptor ring\n");
2491         return -ENOMEM;
2492 }
2493
2494 /**
2495  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2496  * @adapter: board private structure
2497  *
2498  * If this function returns with an error, then it's possible one or
2499  * more of the rings is populated (while the rest are not).  It is the
2500  * callers duty to clean those orphaned rings.
2501  *
2502  * Return 0 on success, negative on failure
2503  **/
2504 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2505 {
2506         int i, err = 0;
2507
2508         for (i = 0; i < adapter->num_tx_queues; i++) {
2509                 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
2510                 if (!err)
2511                         continue;
2512                 hw_dbg(&adapter->hw,
2513                        "Allocation for Tx Queue %u failed\n", i);
2514                 break;
2515         }
2516
2517         return err;
2518 }
2519
2520 /**
2521  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2522  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2523  *
2524  * Returns 0 on success, negative on failure
2525  **/
2526 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
2527 {
2528         int size;
2529
2530         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2531         rx_ring->rx_buffer_info = vzalloc(size);
2532         if (!rx_ring->rx_buffer_info)
2533                 goto err;
2534
2535         /* Round up to nearest 4K */
2536         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2537         rx_ring->size = ALIGN(rx_ring->size, 4096);
2538
2539         rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2540                                            &rx_ring->dma, GFP_KERNEL);
2541
2542         if (!rx_ring->desc)
2543                 goto err;
2544
2545         return 0;
2546 err:
2547         vfree(rx_ring->rx_buffer_info);
2548         rx_ring->rx_buffer_info = NULL;
2549         dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
2550         return -ENOMEM;
2551 }
2552
2553 /**
2554  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2555  * @adapter: board private structure
2556  *
2557  * If this function returns with an error, then it's possible one or
2558  * more of the rings is populated (while the rest are not).  It is the
2559  * callers duty to clean those orphaned rings.
2560  *
2561  * Return 0 on success, negative on failure
2562  **/
2563 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2564 {
2565         int i, err = 0;
2566
2567         for (i = 0; i < adapter->num_rx_queues; i++) {
2568                 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
2569                 if (!err)
2570                         continue;
2571                 hw_dbg(&adapter->hw,
2572                        "Allocation for Rx Queue %u failed\n", i);
2573                 break;
2574         }
2575         return err;
2576 }
2577
2578 /**
2579  * ixgbevf_free_rx_resources - Free Rx Resources
2580  * @rx_ring: ring to clean the resources from
2581  *
2582  * Free all receive software resources
2583  **/
2584 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
2585 {
2586         ixgbevf_clean_rx_ring(rx_ring);
2587
2588         vfree(rx_ring->rx_buffer_info);
2589         rx_ring->rx_buffer_info = NULL;
2590
2591         dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2592                           rx_ring->dma);
2593
2594         rx_ring->desc = NULL;
2595 }
2596
2597 /**
2598  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2599  * @adapter: board private structure
2600  *
2601  * Free all receive software resources
2602  **/
2603 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2604 {
2605         int i;
2606
2607         for (i = 0; i < adapter->num_rx_queues; i++)
2608                 if (adapter->rx_ring[i]->desc)
2609                         ixgbevf_free_rx_resources(adapter->rx_ring[i]);
2610 }
2611
2612 /**
2613  * ixgbevf_open - Called when a network interface is made active
2614  * @netdev: network interface device structure
2615  *
2616  * Returns 0 on success, negative value on failure
2617  *
2618  * The open entry point is called when a network interface is made
2619  * active by the system (IFF_UP).  At this point all resources needed
2620  * for transmit and receive operations are allocated, the interrupt
2621  * handler is registered with the OS, the watchdog timer is started,
2622  * and the stack is notified that the interface is ready.
2623  **/
2624 static int ixgbevf_open(struct net_device *netdev)
2625 {
2626         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2627         struct ixgbe_hw *hw = &adapter->hw;
2628         int err;
2629
2630         /* A previous failure to open the device because of a lack of
2631          * available MSIX vector resources may have reset the number
2632          * of msix vectors variable to zero.  The only way to recover
2633          * is to unload/reload the driver and hope that the system has
2634          * been able to recover some MSIX vector resources.
2635          */
2636         if (!adapter->num_msix_vectors)
2637                 return -ENOMEM;
2638
2639         /* disallow open during test */
2640         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2641                 return -EBUSY;
2642
2643         if (hw->adapter_stopped) {
2644                 ixgbevf_reset(adapter);
2645                 /* if adapter is still stopped then PF isn't up and
2646                  * the vf can't start. */
2647                 if (hw->adapter_stopped) {
2648                         err = IXGBE_ERR_MBX;
2649                         pr_err("Unable to start - perhaps the PF Driver isn't "
2650                                "up yet\n");
2651                         goto err_setup_reset;
2652                 }
2653         }
2654
2655         /* allocate transmit descriptors */
2656         err = ixgbevf_setup_all_tx_resources(adapter);
2657         if (err)
2658                 goto err_setup_tx;
2659
2660         /* allocate receive descriptors */
2661         err = ixgbevf_setup_all_rx_resources(adapter);
2662         if (err)
2663                 goto err_setup_rx;
2664
2665         ixgbevf_configure(adapter);
2666
2667         /*
2668          * Map the Tx/Rx rings to the vectors we were allotted.
2669          * if request_irq will be called in this function map_rings
2670          * must be called *before* up_complete
2671          */
2672         ixgbevf_map_rings_to_vectors(adapter);
2673
2674         ixgbevf_up_complete(adapter);
2675
2676         /* clear any pending interrupts, may auto mask */
2677         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2678         err = ixgbevf_request_irq(adapter);
2679         if (err)
2680                 goto err_req_irq;
2681
2682         ixgbevf_irq_enable(adapter);
2683
2684         return 0;
2685
2686 err_req_irq:
2687         ixgbevf_down(adapter);
2688 err_setup_rx:
2689         ixgbevf_free_all_rx_resources(adapter);
2690 err_setup_tx:
2691         ixgbevf_free_all_tx_resources(adapter);
2692         ixgbevf_reset(adapter);
2693
2694 err_setup_reset:
2695
2696         return err;
2697 }
2698
2699 /**
2700  * ixgbevf_close - Disables a network interface
2701  * @netdev: network interface device structure
2702  *
2703  * Returns 0, this is not allowed to fail
2704  *
2705  * The close entry point is called when an interface is de-activated
2706  * by the OS.  The hardware is still under the drivers control, but
2707  * needs to be disabled.  A global MAC reset is issued to stop the
2708  * hardware, and all transmit and receive resources are freed.
2709  **/
2710 static int ixgbevf_close(struct net_device *netdev)
2711 {
2712         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2713
2714         ixgbevf_down(adapter);
2715         ixgbevf_free_irq(adapter);
2716
2717         ixgbevf_free_all_tx_resources(adapter);
2718         ixgbevf_free_all_rx_resources(adapter);
2719
2720         return 0;
2721 }
2722
2723 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
2724 {
2725         struct net_device *dev = adapter->netdev;
2726
2727         if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
2728                 return;
2729
2730         adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
2731
2732         /* if interface is down do nothing */
2733         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2734             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2735                 return;
2736
2737         /* Hardware has to reinitialize queues and interrupts to
2738          * match packet buffer alignment. Unfortunately, the
2739          * hardware is not flexible enough to do this dynamically.
2740          */
2741         if (netif_running(dev))
2742                 ixgbevf_close(dev);
2743
2744         ixgbevf_clear_interrupt_scheme(adapter);
2745         ixgbevf_init_interrupt_scheme(adapter);
2746
2747         if (netif_running(dev))
2748                 ixgbevf_open(dev);
2749 }
2750
2751 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2752                                 u32 vlan_macip_lens, u32 type_tucmd,
2753                                 u32 mss_l4len_idx)
2754 {
2755         struct ixgbe_adv_tx_context_desc *context_desc;
2756         u16 i = tx_ring->next_to_use;
2757
2758         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2759
2760         i++;
2761         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2762
2763         /* set bits to identify this as an advanced context descriptor */
2764         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2765
2766         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
2767         context_desc->seqnum_seed       = 0;
2768         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
2769         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
2770 }
2771
2772 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2773                        struct ixgbevf_tx_buffer *first,
2774                        u8 *hdr_len)
2775 {
2776         struct sk_buff *skb = first->skb;
2777         u32 vlan_macip_lens, type_tucmd;
2778         u32 mss_l4len_idx, l4len;
2779
2780         if (skb->ip_summed != CHECKSUM_PARTIAL)
2781                 return 0;
2782
2783         if (!skb_is_gso(skb))
2784                 return 0;
2785
2786         if (skb_header_cloned(skb)) {
2787                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2788                 if (err)
2789                         return err;
2790         }
2791
2792         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2793         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2794
2795         if (skb->protocol == htons(ETH_P_IP)) {
2796                 struct iphdr *iph = ip_hdr(skb);
2797                 iph->tot_len = 0;
2798                 iph->check = 0;
2799                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2800                                                          iph->daddr, 0,
2801                                                          IPPROTO_TCP,
2802                                                          0);
2803                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2804                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2805                                    IXGBE_TX_FLAGS_CSUM |
2806                                    IXGBE_TX_FLAGS_IPV4;
2807         } else if (skb_is_gso_v6(skb)) {
2808                 ipv6_hdr(skb)->payload_len = 0;
2809                 tcp_hdr(skb)->check =
2810                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2811                                      &ipv6_hdr(skb)->daddr,
2812                                      0, IPPROTO_TCP, 0);
2813                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2814                                    IXGBE_TX_FLAGS_CSUM;
2815         }
2816
2817         /* compute header lengths */
2818         l4len = tcp_hdrlen(skb);
2819         *hdr_len += l4len;
2820         *hdr_len = skb_transport_offset(skb) + l4len;
2821
2822         /* update gso size and bytecount with header size */
2823         first->gso_segs = skb_shinfo(skb)->gso_segs;
2824         first->bytecount += (first->gso_segs - 1) * *hdr_len;
2825
2826         /* mss_l4len_id: use 1 as index for TSO */
2827         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2828         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2829         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2830
2831         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2832         vlan_macip_lens = skb_network_header_len(skb);
2833         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2834         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2835
2836         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2837                             type_tucmd, mss_l4len_idx);
2838
2839         return 1;
2840 }
2841
2842 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2843                             struct ixgbevf_tx_buffer *first)
2844 {
2845         struct sk_buff *skb = first->skb;
2846         u32 vlan_macip_lens = 0;
2847         u32 mss_l4len_idx = 0;
2848         u32 type_tucmd = 0;
2849
2850         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2851                 u8 l4_hdr = 0;
2852                 switch (skb->protocol) {
2853                 case __constant_htons(ETH_P_IP):
2854                         vlan_macip_lens |= skb_network_header_len(skb);
2855                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2856                         l4_hdr = ip_hdr(skb)->protocol;
2857                         break;
2858                 case __constant_htons(ETH_P_IPV6):
2859                         vlan_macip_lens |= skb_network_header_len(skb);
2860                         l4_hdr = ipv6_hdr(skb)->nexthdr;
2861                         break;
2862                 default:
2863                         if (unlikely(net_ratelimit())) {
2864                                 dev_warn(tx_ring->dev,
2865                                  "partial checksum but proto=%x!\n",
2866                                  first->protocol);
2867                         }
2868                         break;
2869                 }
2870
2871                 switch (l4_hdr) {
2872                 case IPPROTO_TCP:
2873                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2874                         mss_l4len_idx = tcp_hdrlen(skb) <<
2875                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2876                         break;
2877                 case IPPROTO_SCTP:
2878                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2879                         mss_l4len_idx = sizeof(struct sctphdr) <<
2880                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2881                         break;
2882                 case IPPROTO_UDP:
2883                         mss_l4len_idx = sizeof(struct udphdr) <<
2884                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2885                         break;
2886                 default:
2887                         if (unlikely(net_ratelimit())) {
2888                                 dev_warn(tx_ring->dev,
2889                                  "partial checksum but l4 proto=%x!\n",
2890                                  l4_hdr);
2891                         }
2892                         break;
2893                 }
2894
2895                 /* update TX checksum flag */
2896                 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
2897         }
2898
2899         /* vlan_macip_lens: MACLEN, VLAN tag */
2900         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2901         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2902
2903         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2904                             type_tucmd, mss_l4len_idx);
2905 }
2906
2907 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
2908 {
2909         /* set type for advanced descriptor with frame checksum insertion */
2910         __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
2911                                       IXGBE_ADVTXD_DCMD_IFCS |
2912                                       IXGBE_ADVTXD_DCMD_DEXT);
2913
2914         /* set HW vlan bit if vlan is present */
2915         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2916                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
2917
2918         /* set segmentation enable bits for TSO/FSO */
2919         if (tx_flags & IXGBE_TX_FLAGS_TSO)
2920                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
2921
2922         return cmd_type;
2923 }
2924
2925 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
2926                                      u32 tx_flags, unsigned int paylen)
2927 {
2928         __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
2929
2930         /* enable L4 checksum for TSO and TX checksum offload */
2931         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2932                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
2933
2934         /* enble IPv4 checksum for TSO */
2935         if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2936                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
2937
2938         /* use index 1 context for TSO/FSO/FCOE */
2939         if (tx_flags & IXGBE_TX_FLAGS_TSO)
2940                 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
2941
2942         /* Check Context must be set if Tx switch is enabled, which it
2943          * always is for case where virtual functions are running
2944          */
2945         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
2946
2947         tx_desc->read.olinfo_status = olinfo_status;
2948 }
2949
2950 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2951                            struct ixgbevf_tx_buffer *first,
2952                            const u8 hdr_len)
2953 {
2954         dma_addr_t dma;
2955         struct sk_buff *skb = first->skb;
2956         struct ixgbevf_tx_buffer *tx_buffer;
2957         union ixgbe_adv_tx_desc *tx_desc;
2958         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
2959         unsigned int data_len = skb->data_len;
2960         unsigned int size = skb_headlen(skb);
2961         unsigned int paylen = skb->len - hdr_len;
2962         u32 tx_flags = first->tx_flags;
2963         __le32 cmd_type;
2964         u16 i = tx_ring->next_to_use;
2965
2966         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2967
2968         ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
2969         cmd_type = ixgbevf_tx_cmd_type(tx_flags);
2970
2971         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2972         if (dma_mapping_error(tx_ring->dev, dma))
2973                 goto dma_error;
2974
2975         /* record length, and DMA address */
2976         dma_unmap_len_set(first, len, size);
2977         dma_unmap_addr_set(first, dma, dma);
2978
2979         tx_desc->read.buffer_addr = cpu_to_le64(dma);
2980
2981         for (;;) {
2982                 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
2983                         tx_desc->read.cmd_type_len =
2984                                 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
2985
2986                         i++;
2987                         tx_desc++;
2988                         if (i == tx_ring->count) {
2989                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2990                                 i = 0;
2991                         }
2992
2993                         dma += IXGBE_MAX_DATA_PER_TXD;
2994                         size -= IXGBE_MAX_DATA_PER_TXD;
2995
2996                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
2997                         tx_desc->read.olinfo_status = 0;
2998                 }
2999
3000                 if (likely(!data_len))
3001                         break;
3002
3003                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
3004
3005                 i++;
3006                 tx_desc++;
3007                 if (i == tx_ring->count) {
3008                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3009                         i = 0;
3010                 }
3011
3012                 size = skb_frag_size(frag);
3013                 data_len -= size;
3014
3015                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3016                                        DMA_TO_DEVICE);
3017                 if (dma_mapping_error(tx_ring->dev, dma))
3018                         goto dma_error;
3019
3020                 tx_buffer = &tx_ring->tx_buffer_info[i];
3021                 dma_unmap_len_set(tx_buffer, len, size);
3022                 dma_unmap_addr_set(tx_buffer, dma, dma);
3023
3024                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3025                 tx_desc->read.olinfo_status = 0;
3026
3027                 frag++;
3028         }
3029
3030         /* write last descriptor with RS and EOP bits */
3031         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3032         tx_desc->read.cmd_type_len = cmd_type;
3033
3034         /* set the timestamp */
3035         first->time_stamp = jiffies;
3036
3037         /* Force memory writes to complete before letting h/w know there
3038          * are new descriptors to fetch.  (Only applicable for weak-ordered
3039          * memory model archs, such as IA-64).
3040          *
3041          * We also need this memory barrier (wmb) to make certain all of the
3042          * status bits have been updated before next_to_watch is written.
3043          */
3044         wmb();
3045
3046         /* set next_to_watch value indicating a packet is present */
3047         first->next_to_watch = tx_desc;
3048
3049         i++;
3050         if (i == tx_ring->count)
3051                 i = 0;
3052
3053         tx_ring->next_to_use = i;
3054
3055         /* notify HW of packet */
3056         writel(i, tx_ring->tail);
3057
3058         return;
3059 dma_error:
3060         dev_err(tx_ring->dev, "TX DMA map failed\n");
3061
3062         /* clear dma mappings for failed tx_buffer_info map */
3063         for (;;) {
3064                 tx_buffer = &tx_ring->tx_buffer_info[i];
3065                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3066                 if (tx_buffer == first)
3067                         break;
3068                 if (i == 0)
3069                         i = tx_ring->count;
3070                 i--;
3071         }
3072
3073         tx_ring->next_to_use = i;
3074 }
3075
3076 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3077 {
3078         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
3079         /* Herbert's original patch had:
3080          *  smp_mb__after_netif_stop_queue();
3081          * but since that doesn't exist yet, just open code it. */
3082         smp_mb();
3083
3084         /* We need to check again in a case another CPU has just
3085          * made room available. */
3086         if (likely(ixgbevf_desc_unused(tx_ring) < size))
3087                 return -EBUSY;
3088
3089         /* A reprieve! - use start_queue because it doesn't call schedule */
3090         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
3091         ++tx_ring->tx_stats.restart_queue;
3092
3093         return 0;
3094 }
3095
3096 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3097 {
3098         if (likely(ixgbevf_desc_unused(tx_ring) >= size))
3099                 return 0;
3100         return __ixgbevf_maybe_stop_tx(tx_ring, size);
3101 }
3102
3103 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3104 {
3105         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3106         struct ixgbevf_tx_buffer *first;
3107         struct ixgbevf_ring *tx_ring;
3108         int tso;
3109         u32 tx_flags = 0;
3110         u16 count = TXD_USE_COUNT(skb_headlen(skb));
3111 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3112         unsigned short f;
3113 #endif
3114         u8 hdr_len = 0;
3115         u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
3116
3117         if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
3118                 dev_kfree_skb(skb);
3119                 return NETDEV_TX_OK;
3120         }
3121
3122         tx_ring = adapter->tx_ring[skb->queue_mapping];
3123
3124         /*
3125          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3126          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3127          *       + 2 desc gap to keep tail from touching head,
3128          *       + 1 desc for context descriptor,
3129          * otherwise try next time
3130          */
3131 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3132         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3133                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3134 #else
3135         count += skb_shinfo(skb)->nr_frags;
3136 #endif
3137         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3138                 tx_ring->tx_stats.tx_busy++;
3139                 return NETDEV_TX_BUSY;
3140         }
3141
3142         /* record the location of the first descriptor for this packet */
3143         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3144         first->skb = skb;
3145         first->bytecount = skb->len;
3146         first->gso_segs = 1;
3147
3148         if (vlan_tx_tag_present(skb)) {
3149                 tx_flags |= vlan_tx_tag_get(skb);
3150                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3151                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3152         }
3153
3154         /* record initial flags and protocol */
3155         first->tx_flags = tx_flags;
3156         first->protocol = vlan_get_protocol(skb);
3157
3158         tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3159         if (tso < 0)
3160                 goto out_drop;
3161         else if (!tso)
3162                 ixgbevf_tx_csum(tx_ring, first);
3163
3164         ixgbevf_tx_map(tx_ring, first, hdr_len);
3165
3166         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
3167
3168         return NETDEV_TX_OK;
3169
3170 out_drop:
3171         dev_kfree_skb_any(first->skb);
3172         first->skb = NULL;
3173
3174         return NETDEV_TX_OK;
3175 }
3176
3177 /**
3178  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3179  * @netdev: network interface device structure
3180  * @p: pointer to an address structure
3181  *
3182  * Returns 0 on success, negative on failure
3183  **/
3184 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3185 {
3186         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3187         struct ixgbe_hw *hw = &adapter->hw;
3188         struct sockaddr *addr = p;
3189
3190         if (!is_valid_ether_addr(addr->sa_data))
3191                 return -EADDRNOTAVAIL;
3192
3193         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3194         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3195
3196         spin_lock_bh(&adapter->mbx_lock);
3197
3198         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3199
3200         spin_unlock_bh(&adapter->mbx_lock);
3201
3202         return 0;
3203 }
3204
3205 /**
3206  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3207  * @netdev: network interface device structure
3208  * @new_mtu: new value for maximum frame size
3209  *
3210  * Returns 0 on success, negative on failure
3211  **/
3212 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3213 {
3214         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3215         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3216         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3217
3218         switch (adapter->hw.api_version) {
3219         case ixgbe_mbox_api_11:
3220                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3221                 break;
3222         default:
3223                 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3224                         max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3225                 break;
3226         }
3227
3228         /* MTU < 68 is an error and causes problems on some kernels */
3229         if ((new_mtu < 68) || (max_frame > max_possible_frame))
3230                 return -EINVAL;
3231
3232         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3233                netdev->mtu, new_mtu);
3234         /* must set new MTU before calling down or up */
3235         netdev->mtu = new_mtu;
3236
3237         if (netif_running(netdev))
3238                 ixgbevf_reinit_locked(adapter);
3239
3240         return 0;
3241 }
3242
3243 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
3244 {
3245         struct net_device *netdev = pci_get_drvdata(pdev);
3246         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3247 #ifdef CONFIG_PM
3248         int retval = 0;
3249 #endif
3250
3251         netif_device_detach(netdev);
3252
3253         if (netif_running(netdev)) {
3254                 rtnl_lock();
3255                 ixgbevf_down(adapter);
3256                 ixgbevf_free_irq(adapter);
3257                 ixgbevf_free_all_tx_resources(adapter);
3258                 ixgbevf_free_all_rx_resources(adapter);
3259                 rtnl_unlock();
3260         }
3261
3262         ixgbevf_clear_interrupt_scheme(adapter);
3263
3264 #ifdef CONFIG_PM
3265         retval = pci_save_state(pdev);
3266         if (retval)
3267                 return retval;
3268
3269 #endif
3270         pci_disable_device(pdev);
3271
3272         return 0;
3273 }
3274
3275 #ifdef CONFIG_PM
3276 static int ixgbevf_resume(struct pci_dev *pdev)
3277 {
3278         struct net_device *netdev = pci_get_drvdata(pdev);
3279         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3280         u32 err;
3281
3282         pci_set_power_state(pdev, PCI_D0);
3283         pci_restore_state(pdev);
3284         /*
3285          * pci_restore_state clears dev->state_saved so call
3286          * pci_save_state to restore it.
3287          */
3288         pci_save_state(pdev);
3289
3290         err = pci_enable_device_mem(pdev);
3291         if (err) {
3292                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3293                 return err;
3294         }
3295         pci_set_master(pdev);
3296
3297         ixgbevf_reset(adapter);
3298
3299         rtnl_lock();
3300         err = ixgbevf_init_interrupt_scheme(adapter);
3301         rtnl_unlock();
3302         if (err) {
3303                 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3304                 return err;
3305         }
3306
3307         if (netif_running(netdev)) {
3308                 err = ixgbevf_open(netdev);
3309                 if (err)
3310                         return err;
3311         }
3312
3313         netif_device_attach(netdev);
3314
3315         return err;
3316 }
3317
3318 #endif /* CONFIG_PM */
3319 static void ixgbevf_shutdown(struct pci_dev *pdev)
3320 {
3321         ixgbevf_suspend(pdev, PMSG_SUSPEND);
3322 }
3323
3324 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3325                                                 struct rtnl_link_stats64 *stats)
3326 {
3327         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3328         unsigned int start;
3329         u64 bytes, packets;
3330         const struct ixgbevf_ring *ring;
3331         int i;
3332
3333         ixgbevf_update_stats(adapter);
3334
3335         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3336
3337         for (i = 0; i < adapter->num_rx_queues; i++) {
3338                 ring = adapter->rx_ring[i];
3339                 do {
3340                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3341                         bytes = ring->stats.bytes;
3342                         packets = ring->stats.packets;
3343                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3344                 stats->rx_bytes += bytes;
3345                 stats->rx_packets += packets;
3346         }
3347
3348         for (i = 0; i < adapter->num_tx_queues; i++) {
3349                 ring = adapter->tx_ring[i];
3350                 do {
3351                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3352                         bytes = ring->stats.bytes;
3353                         packets = ring->stats.packets;
3354                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3355                 stats->tx_bytes += bytes;
3356                 stats->tx_packets += packets;
3357         }
3358
3359         return stats;
3360 }
3361
3362 static const struct net_device_ops ixgbevf_netdev_ops = {
3363         .ndo_open               = ixgbevf_open,
3364         .ndo_stop               = ixgbevf_close,
3365         .ndo_start_xmit         = ixgbevf_xmit_frame,
3366         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3367         .ndo_get_stats64        = ixgbevf_get_stats,
3368         .ndo_validate_addr      = eth_validate_addr,
3369         .ndo_set_mac_address    = ixgbevf_set_mac,
3370         .ndo_change_mtu         = ixgbevf_change_mtu,
3371         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3372         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3373         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3374 #ifdef CONFIG_NET_RX_BUSY_POLL
3375         .ndo_busy_poll          = ixgbevf_busy_poll_recv,
3376 #endif
3377 };
3378
3379 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3380 {
3381         dev->netdev_ops = &ixgbevf_netdev_ops;
3382         ixgbevf_set_ethtool_ops(dev);
3383         dev->watchdog_timeo = 5 * HZ;
3384 }
3385
3386 /**
3387  * ixgbevf_probe - Device Initialization Routine
3388  * @pdev: PCI device information struct
3389  * @ent: entry in ixgbevf_pci_tbl
3390  *
3391  * Returns 0 on success, negative on failure
3392  *
3393  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3394  * The OS initialization, configuring of the adapter private structure,
3395  * and a hardware reset occur.
3396  **/
3397 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3398 {
3399         struct net_device *netdev;
3400         struct ixgbevf_adapter *adapter = NULL;
3401         struct ixgbe_hw *hw = NULL;
3402         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3403         static int cards_found;
3404         int err, pci_using_dac;
3405
3406         err = pci_enable_device(pdev);
3407         if (err)
3408                 return err;
3409
3410         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
3411                 pci_using_dac = 1;
3412         } else {
3413                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3414                 if (err) {
3415                         dev_err(&pdev->dev, "No usable DMA "
3416                                 "configuration, aborting\n");
3417                         goto err_dma;
3418                 }
3419                 pci_using_dac = 0;
3420         }
3421
3422         err = pci_request_regions(pdev, ixgbevf_driver_name);
3423         if (err) {
3424                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3425                 goto err_pci_reg;
3426         }
3427
3428         pci_set_master(pdev);
3429
3430         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3431                                    MAX_TX_QUEUES);
3432         if (!netdev) {
3433                 err = -ENOMEM;
3434                 goto err_alloc_etherdev;
3435         }
3436
3437         SET_NETDEV_DEV(netdev, &pdev->dev);
3438
3439         pci_set_drvdata(pdev, netdev);
3440         adapter = netdev_priv(netdev);
3441
3442         adapter->netdev = netdev;
3443         adapter->pdev = pdev;
3444         hw = &adapter->hw;
3445         hw->back = adapter;
3446         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3447
3448         /*
3449          * call save state here in standalone driver because it relies on
3450          * adapter struct to exist, and needs to call netdev_priv
3451          */
3452         pci_save_state(pdev);
3453
3454         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3455                               pci_resource_len(pdev, 0));
3456         if (!hw->hw_addr) {
3457                 err = -EIO;
3458                 goto err_ioremap;
3459         }
3460
3461         ixgbevf_assign_netdev_ops(netdev);
3462
3463         adapter->bd_number = cards_found;
3464
3465         /* Setup hw api */
3466         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3467         hw->mac.type  = ii->mac;
3468
3469         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3470                sizeof(struct ixgbe_mbx_operations));
3471
3472         /* setup the private structure */
3473         err = ixgbevf_sw_init(adapter);
3474         if (err)
3475                 goto err_sw_init;
3476
3477         /* The HW MAC address was set and/or determined in sw_init */
3478         if (!is_valid_ether_addr(netdev->dev_addr)) {
3479                 pr_err("invalid MAC address\n");
3480                 err = -EIO;
3481                 goto err_sw_init;
3482         }
3483
3484         netdev->hw_features = NETIF_F_SG |
3485                            NETIF_F_IP_CSUM |
3486                            NETIF_F_IPV6_CSUM |
3487                            NETIF_F_TSO |
3488                            NETIF_F_TSO6 |
3489                            NETIF_F_RXCSUM;
3490
3491         netdev->features = netdev->hw_features |
3492                            NETIF_F_HW_VLAN_CTAG_TX |
3493                            NETIF_F_HW_VLAN_CTAG_RX |
3494                            NETIF_F_HW_VLAN_CTAG_FILTER;
3495
3496         netdev->vlan_features |= NETIF_F_TSO;
3497         netdev->vlan_features |= NETIF_F_TSO6;
3498         netdev->vlan_features |= NETIF_F_IP_CSUM;
3499         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3500         netdev->vlan_features |= NETIF_F_SG;
3501
3502         if (pci_using_dac)
3503                 netdev->features |= NETIF_F_HIGHDMA;
3504
3505         netdev->priv_flags |= IFF_UNICAST_FLT;
3506
3507         init_timer(&adapter->watchdog_timer);
3508         adapter->watchdog_timer.function = ixgbevf_watchdog;
3509         adapter->watchdog_timer.data = (unsigned long)adapter;
3510
3511         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3512         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3513
3514         err = ixgbevf_init_interrupt_scheme(adapter);
3515         if (err)
3516                 goto err_sw_init;
3517
3518         strcpy(netdev->name, "eth%d");
3519
3520         err = register_netdev(netdev);
3521         if (err)
3522                 goto err_register;
3523
3524         netif_carrier_off(netdev);
3525
3526         ixgbevf_init_last_counter_stats(adapter);
3527
3528         /* print the MAC address */
3529         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3530
3531         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3532
3533         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3534         cards_found++;
3535         return 0;
3536
3537 err_register:
3538         ixgbevf_clear_interrupt_scheme(adapter);
3539 err_sw_init:
3540         ixgbevf_reset_interrupt_capability(adapter);
3541         iounmap(hw->hw_addr);
3542 err_ioremap:
3543         free_netdev(netdev);
3544 err_alloc_etherdev:
3545         pci_release_regions(pdev);
3546 err_pci_reg:
3547 err_dma:
3548         pci_disable_device(pdev);
3549         return err;
3550 }
3551
3552 /**
3553  * ixgbevf_remove - Device Removal Routine
3554  * @pdev: PCI device information struct
3555  *
3556  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3557  * that it should release a PCI device.  The could be caused by a
3558  * Hot-Plug event, or because the driver is going to be removed from
3559  * memory.
3560  **/
3561 static void ixgbevf_remove(struct pci_dev *pdev)
3562 {
3563         struct net_device *netdev = pci_get_drvdata(pdev);
3564         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3565
3566         set_bit(__IXGBEVF_DOWN, &adapter->state);
3567
3568         del_timer_sync(&adapter->watchdog_timer);
3569
3570         cancel_work_sync(&adapter->reset_task);
3571         cancel_work_sync(&adapter->watchdog_task);
3572
3573         if (netdev->reg_state == NETREG_REGISTERED)
3574                 unregister_netdev(netdev);
3575
3576         ixgbevf_clear_interrupt_scheme(adapter);
3577         ixgbevf_reset_interrupt_capability(adapter);
3578
3579         iounmap(adapter->hw.hw_addr);
3580         pci_release_regions(pdev);
3581
3582         hw_dbg(&adapter->hw, "Remove complete\n");
3583
3584         free_netdev(netdev);
3585
3586         pci_disable_device(pdev);
3587 }
3588
3589 /**
3590  * ixgbevf_io_error_detected - called when PCI error is detected
3591  * @pdev: Pointer to PCI device
3592  * @state: The current pci connection state
3593  *
3594  * This function is called after a PCI bus error affecting
3595  * this device has been detected.
3596  */
3597 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3598                                                   pci_channel_state_t state)
3599 {
3600         struct net_device *netdev = pci_get_drvdata(pdev);
3601         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3602
3603         netif_device_detach(netdev);
3604
3605         if (state == pci_channel_io_perm_failure)
3606                 return PCI_ERS_RESULT_DISCONNECT;
3607
3608         if (netif_running(netdev))
3609                 ixgbevf_down(adapter);
3610
3611         pci_disable_device(pdev);
3612
3613         /* Request a slot slot reset. */
3614         return PCI_ERS_RESULT_NEED_RESET;
3615 }
3616
3617 /**
3618  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3619  * @pdev: Pointer to PCI device
3620  *
3621  * Restart the card from scratch, as if from a cold-boot. Implementation
3622  * resembles the first-half of the ixgbevf_resume routine.
3623  */
3624 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3625 {
3626         struct net_device *netdev = pci_get_drvdata(pdev);
3627         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3628
3629         if (pci_enable_device_mem(pdev)) {
3630                 dev_err(&pdev->dev,
3631                         "Cannot re-enable PCI device after reset.\n");
3632                 return PCI_ERS_RESULT_DISCONNECT;
3633         }
3634
3635         pci_set_master(pdev);
3636
3637         ixgbevf_reset(adapter);
3638
3639         return PCI_ERS_RESULT_RECOVERED;
3640 }
3641
3642 /**
3643  * ixgbevf_io_resume - called when traffic can start flowing again.
3644  * @pdev: Pointer to PCI device
3645  *
3646  * This callback is called when the error recovery driver tells us that
3647  * its OK to resume normal operation. Implementation resembles the
3648  * second-half of the ixgbevf_resume routine.
3649  */
3650 static void ixgbevf_io_resume(struct pci_dev *pdev)
3651 {
3652         struct net_device *netdev = pci_get_drvdata(pdev);
3653         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3654
3655         if (netif_running(netdev))
3656                 ixgbevf_up(adapter);
3657
3658         netif_device_attach(netdev);
3659 }
3660
3661 /* PCI Error Recovery (ERS) */
3662 static const struct pci_error_handlers ixgbevf_err_handler = {
3663         .error_detected = ixgbevf_io_error_detected,
3664         .slot_reset = ixgbevf_io_slot_reset,
3665         .resume = ixgbevf_io_resume,
3666 };
3667
3668 static struct pci_driver ixgbevf_driver = {
3669         .name     = ixgbevf_driver_name,
3670         .id_table = ixgbevf_pci_tbl,
3671         .probe    = ixgbevf_probe,
3672         .remove   = ixgbevf_remove,
3673 #ifdef CONFIG_PM
3674         /* Power Management Hooks */
3675         .suspend  = ixgbevf_suspend,
3676         .resume   = ixgbevf_resume,
3677 #endif
3678         .shutdown = ixgbevf_shutdown,
3679         .err_handler = &ixgbevf_err_handler
3680 };
3681
3682 /**
3683  * ixgbevf_init_module - Driver Registration Routine
3684  *
3685  * ixgbevf_init_module is the first routine called when the driver is
3686  * loaded. All it does is register with the PCI subsystem.
3687  **/
3688 static int __init ixgbevf_init_module(void)
3689 {
3690         int ret;
3691         pr_info("%s - version %s\n", ixgbevf_driver_string,
3692                 ixgbevf_driver_version);
3693
3694         pr_info("%s\n", ixgbevf_copyright);
3695
3696         ret = pci_register_driver(&ixgbevf_driver);
3697         return ret;
3698 }
3699
3700 module_init(ixgbevf_init_module);
3701
3702 /**
3703  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3704  *
3705  * ixgbevf_exit_module is called just before the driver is removed
3706  * from memory.
3707  **/
3708 static void __exit ixgbevf_exit_module(void)
3709 {
3710         pci_unregister_driver(&ixgbevf_driver);
3711 }
3712
3713 #ifdef DEBUG
3714 /**
3715  * ixgbevf_get_hw_dev_name - return device name string
3716  * used by hardware layer to print debugging information
3717  **/
3718 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3719 {
3720         struct ixgbevf_adapter *adapter = hw->back;
3721         return adapter->netdev->name;
3722 }
3723
3724 #endif
3725 module_exit(ixgbevf_exit_module);
3726
3727 /* ixgbevf_main.c */