drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 /* Local includes */
  28 #include "i40e.h"
  29 #include "i40e_diag.h"
  30 #ifdef CONFIG_I40E_VXLAN
  31 #include <net/vxlan.h>
  32 #endif
  33
  34 const char i40e_driver_name[] = "i40e";
  35 static const char i40e_driver_string[] =
  36                         "Intel(R) Ethernet Connection XL710 Network Driver";
  37
  38 #define DRV_KERN "-k"
  39
  40 #define DRV_VERSION_MAJOR 1
  41 #define DRV_VERSION_MINOR 0
  42 #define DRV_VERSION_BUILD 11
  43 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  44              __stringify(DRV_VERSION_MINOR) "." \
  45              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  46 const char i40e_driver_version_str[] = DRV_VERSION;
  47 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  48
  49 /* a bit of forward declarations */
  50 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  51 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  52 static int i40e_add_vsi(struct i40e_vsi *vsi);
  53 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  54 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  55 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  56 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  57 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  58 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  59 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  60
  61 /* i40e_pci_tbl - PCI Device ID Table
  62  *
  63  * Last entry must be all 0s
  64  *
  65  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  66  *   Class, Class Mask, private data (not used) }
  67  */
  68 static const struct pci_device_id i40e_pci_tbl[] = {
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_A), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  78         /* required last entry */
  79         {0, }
  80 };
  81 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  82
  83 #define I40E_MAX_VF_COUNT 128
  84 static int debug = -1;
  85 module_param(debug, int, 0);
  86 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  87
  88 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  89 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  90 MODULE_LICENSE("GPL");
  91 MODULE_VERSION(DRV_VERSION);
  92
  93 /**
  94  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  95  * @hw:   pointer to the HW structure
  96  * @mem:  ptr to mem struct to fill out
  97  * @size: size of memory requested
  98  * @alignment: what to align the allocation to
  99  **/
 100 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 101                             u64 size, u32 alignment)
 102 {
 103         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 104
 105         mem->size = ALIGN(size, alignment);
 106         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 107                                       &mem->pa, GFP_KERNEL);
 108         if (!mem->va)
 109                 return -ENOMEM;
 110
 111         return 0;
 112 }
 113
 114 /**
 115  * i40e_free_dma_mem_d - OS specific memory free for shared code
 116  * @hw:   pointer to the HW structure
 117  * @mem:  ptr to mem struct to free
 118  **/
 119 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 120 {
 121         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 122
 123         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 124         mem->va = NULL;
 125         mem->pa = 0;
 126         mem->size = 0;
 127
 128         return 0;
 129 }
 130
 131 /**
 132  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 133  * @hw:   pointer to the HW structure
 134  * @mem:  ptr to mem struct to fill out
 135  * @size: size of memory requested
 136  **/
 137 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 138                              u32 size)
 139 {
 140         mem->size = size;
 141         mem->va = kzalloc(size, GFP_KERNEL);
 142
 143         if (!mem->va)
 144                 return -ENOMEM;
 145
 146         return 0;
 147 }
 148
 149 /**
 150  * i40e_free_virt_mem_d - OS specific memory free for shared code
 151  * @hw:   pointer to the HW structure
 152  * @mem:  ptr to mem struct to free
 153  **/
 154 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 155 {
 156         /* it's ok to kfree a NULL pointer */
 157         kfree(mem->va);
 158         mem->va = NULL;
 159         mem->size = 0;
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * i40e_get_lump - find a lump of free generic resource
 166  * @pf: board private structure
 167  * @pile: the pile of resource to search
 168  * @needed: the number of items needed
 169  * @id: an owner id to stick on the items assigned
 170  *
 171  * Returns the base item index of the lump, or negative for error
 172  *
 173  * The search_hint trick and lack of advanced fit-finding only work
 174  * because we're highly likely to have all the same size lump requests.
 175  * Linear search time and any fragmentation should be minimal.
 176  **/
 177 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 178                          u16 needed, u16 id)
 179 {
 180         int ret = -ENOMEM;
 181         int i, j;
 182
 183         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 184                 dev_info(&pf->pdev->dev,
 185                          "param err: pile=%p needed=%d id=0x%04x\n",
 186                          pile, needed, id);
 187                 return -EINVAL;
 188         }
 189
 190         /* start the linear search with an imperfect hint */
 191         i = pile->search_hint;
 192         while (i < pile->num_entries) {
 193                 /* skip already allocated entries */
 194                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 195                         i++;
 196                         continue;
 197                 }
 198
 199                 /* do we have enough in this lump? */
 200                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 201                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 202                                 break;
 203                 }
 204
 205                 if (j == needed) {
 206                         /* there was enough, so assign it to the requestor */
 207                         for (j = 0; j < needed; j++)
 208                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 209                         ret = i;
 210                         pile->search_hint = i + j;
 211                         break;
 212                 } else {
 213                         /* not enough, so skip over it and continue looking */
 214                         i += j;
 215                 }
 216         }
 217
 218         return ret;
 219 }
 220
 221 /**
 222  * i40e_put_lump - return a lump of generic resource
 223  * @pile: the pile of resource to search
 224  * @index: the base item index
 225  * @id: the owner id of the items assigned
 226  *
 227  * Returns the count of items in the lump
 228  **/
 229 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 230 {
 231         int valid_id = (id | I40E_PILE_VALID_BIT);
 232         int count = 0;
 233         int i;
 234
 235         if (!pile || index >= pile->num_entries)
 236                 return -EINVAL;
 237
 238         for (i = index;
 239              i < pile->num_entries && pile->list[i] == valid_id;
 240              i++) {
 241                 pile->list[i] = 0;
 242                 count++;
 243         }
 244
 245         if (count && index < pile->search_hint)
 246                 pile->search_hint = index;
 247
 248         return count;
 249 }
 250
 251 /**
 252  * i40e_service_event_schedule - Schedule the service task to wake up
 253  * @pf: board private structure
 254  *
 255  * If not already scheduled, this puts the task into the work queue
 256  **/
 257 static void i40e_service_event_schedule(struct i40e_pf *pf)
 258 {
 259         if (!test_bit(__I40E_DOWN, &pf->state) &&
 260             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 261             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 262                 schedule_work(&pf->service_task);
 263 }
 264
 265 /**
 266  * i40e_tx_timeout - Respond to a Tx Hang
 267  * @netdev: network interface device structure
 268  *
 269  * If any port has noticed a Tx timeout, it is likely that the whole
 270  * device is munged, not just the one netdev port, so go for the full
 271  * reset.
 272  **/
 273 #ifdef I40E_FCOE
 274 void i40e_tx_timeout(struct net_device *netdev)
 275 #else
 276 static void i40e_tx_timeout(struct net_device *netdev)
 277 #endif
 278 {
 279         struct i40e_netdev_priv *np = netdev_priv(netdev);
 280         struct i40e_vsi *vsi = np->vsi;
 281         struct i40e_pf *pf = vsi->back;
 282
 283         pf->tx_timeout_count++;
 284
 285         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 286                 pf->tx_timeout_recovery_level = 1;
 287         pf->tx_timeout_last_recovery = jiffies;
 288         netdev_info(netdev, "tx_timeout recovery level %d\n",
 289                     pf->tx_timeout_recovery_level);
 290
 291         switch (pf->tx_timeout_recovery_level) {
 292         case 0:
 293                 /* disable and re-enable queues for the VSI */
 294                 if (in_interrupt()) {
 295                         set_bit(__I40E_REINIT_REQUESTED, &pf->state);
 296                         set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
 297                 } else {
 298                         i40e_vsi_reinit_locked(vsi);
 299                 }
 300                 break;
 301         case 1:
 302                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 303                 break;
 304         case 2:
 305                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 306                 break;
 307         case 3:
 308                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 309                 break;
 310         default:
 311                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 312                 set_bit(__I40E_DOWN_REQUESTED, &pf->state);
 313                 set_bit(__I40E_DOWN_REQUESTED, &vsi->state);
 314                 break;
 315         }
 316         i40e_service_event_schedule(pf);
 317         pf->tx_timeout_recovery_level++;
 318 }
 319
 320 /**
 321  * i40e_release_rx_desc - Store the new tail and head values
 322  * @rx_ring: ring to bump
 323  * @val: new head index
 324  **/
 325 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 326 {
 327         rx_ring->next_to_use = val;
 328
 329         /* Force memory writes to complete before letting h/w
 330          * know there are new descriptors to fetch.  (Only
 331          * applicable for weak-ordered memory model archs,
 332          * such as IA-64).
 333          */
 334         wmb();
 335         writel(val, rx_ring->tail);
 336 }
 337
 338 /**
 339  * i40e_get_vsi_stats_struct - Get System Network Statistics
 340  * @vsi: the VSI we care about
 341  *
 342  * Returns the address of the device statistics structure.
 343  * The statistics are actually updated from the service task.
 344  **/
 345 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 346 {
 347         return &vsi->net_stats;
 348 }
 349
 350 /**
 351  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 352  * @netdev: network interface device structure
 353  *
 354  * Returns the address of the device statistics structure.
 355  * The statistics are actually updated from the service task.
 356  **/
 357 #ifdef I40E_FCOE
 358 struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 359                                              struct net_device *netdev,
 360                                              struct rtnl_link_stats64 *stats)
 361 #else
 362 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 363                                              struct net_device *netdev,
 364                                              struct rtnl_link_stats64 *stats)
 365 #endif
 366 {
 367         struct i40e_netdev_priv *np = netdev_priv(netdev);
 368         struct i40e_ring *tx_ring, *rx_ring;
 369         struct i40e_vsi *vsi = np->vsi;
 370         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 371         int i;
 372
 373         if (test_bit(__I40E_DOWN, &vsi->state))
 374                 return stats;
 375
 376         if (!vsi->tx_rings)
 377                 return stats;
 378
 379         rcu_read_lock();
 380         for (i = 0; i < vsi->num_queue_pairs; i++) {
 381                 u64 bytes, packets;
 382                 unsigned int start;
 383
 384                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 385                 if (!tx_ring)
 386                         continue;
 387
 388                 do {
 389                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
 390                         packets = tx_ring->stats.packets;
 391                         bytes   = tx_ring->stats.bytes;
 392                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
 393
 394                 stats->tx_packets += packets;
 395                 stats->tx_bytes   += bytes;
 396                 rx_ring = &tx_ring[1];
 397
 398                 do {
 399                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 400                         packets = rx_ring->stats.packets;
 401                         bytes   = rx_ring->stats.bytes;
 402                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 403
 404                 stats->rx_packets += packets;
 405                 stats->rx_bytes   += bytes;
 406         }
 407         rcu_read_unlock();
 408
 409         /* following stats updated by i40e_watchdog_subtask() */
 410         stats->multicast        = vsi_stats->multicast;
 411         stats->tx_errors        = vsi_stats->tx_errors;
 412         stats->tx_dropped       = vsi_stats->tx_dropped;
 413         stats->rx_errors        = vsi_stats->rx_errors;
 414         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 415         stats->rx_length_errors = vsi_stats->rx_length_errors;
 416
 417         return stats;
 418 }
 419
 420 /**
 421  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 422  * @vsi: the VSI to have its stats reset
 423  **/
 424 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 425 {
 426         struct rtnl_link_stats64 *ns;
 427         int i;
 428
 429         if (!vsi)
 430                 return;
 431
 432         ns = i40e_get_vsi_stats_struct(vsi);
 433         memset(ns, 0, sizeof(*ns));
 434         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 435         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 436         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 437         if (vsi->rx_rings && vsi->rx_rings[0]) {
 438                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 439                         memset(&vsi->rx_rings[i]->stats, 0 ,
 440                                sizeof(vsi->rx_rings[i]->stats));
 441                         memset(&vsi->rx_rings[i]->rx_stats, 0 ,
 442                                sizeof(vsi->rx_rings[i]->rx_stats));
 443                         memset(&vsi->tx_rings[i]->stats, 0 ,
 444                                sizeof(vsi->tx_rings[i]->stats));
 445                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 446                                sizeof(vsi->tx_rings[i]->tx_stats));
 447                 }
 448         }
 449         vsi->stat_offsets_loaded = false;
 450 }
 451
 452 /**
 453  * i40e_pf_reset_stats - Reset all of the stats for the given pf
 454  * @pf: the PF to be reset
 455  **/
 456 void i40e_pf_reset_stats(struct i40e_pf *pf)
 457 {
 458         int i;
 459
 460         memset(&pf->stats, 0, sizeof(pf->stats));
 461         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 462         pf->stat_offsets_loaded = false;
 463
 464         for (i = 0; i < I40E_MAX_VEB; i++) {
 465                 if (pf->veb[i]) {
 466                         memset(&pf->veb[i]->stats, 0,
 467                                sizeof(pf->veb[i]->stats));
 468                         memset(&pf->veb[i]->stats_offsets, 0,
 469                                sizeof(pf->veb[i]->stats_offsets));
 470                         pf->veb[i]->stat_offsets_loaded = false;
 471                 }
 472         }
 473 }
 474
 475 /**
 476  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 477  * @hw: ptr to the hardware info
 478  * @hireg: the high 32 bit reg to read
 479  * @loreg: the low 32 bit reg to read
 480  * @offset_loaded: has the initial offset been loaded yet
 481  * @offset: ptr to current offset value
 482  * @stat: ptr to the stat
 483  *
 484  * Since the device stats are not reset at PFReset, they likely will not
 485  * be zeroed when the driver starts.  We'll save the first values read
 486  * and use them as offsets to be subtracted from the raw values in order
 487  * to report stats that count from zero.  In the process, we also manage
 488  * the potential roll-over.
 489  **/
 490 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 491                                bool offset_loaded, u64 *offset, u64 *stat)
 492 {
 493         u64 new_data;
 494
 495         if (hw->device_id == I40E_DEV_ID_QEMU) {
 496                 new_data = rd32(hw, loreg);
 497                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 498         } else {
 499                 new_data = rd64(hw, loreg);
 500         }
 501         if (!offset_loaded)
 502                 *offset = new_data;
 503         if (likely(new_data >= *offset))
 504                 *stat = new_data - *offset;
 505         else
 506                 *stat = (new_data + ((u64)1 << 48)) - *offset;
 507         *stat &= 0xFFFFFFFFFFFFULL;
 508 }
 509
 510 /**
 511  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 512  * @hw: ptr to the hardware info
 513  * @reg: the hw reg to read
 514  * @offset_loaded: has the initial offset been loaded yet
 515  * @offset: ptr to current offset value
 516  * @stat: ptr to the stat
 517  **/
 518 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 519                                bool offset_loaded, u64 *offset, u64 *stat)
 520 {
 521         u32 new_data;
 522
 523         new_data = rd32(hw, reg);
 524         if (!offset_loaded)
 525                 *offset = new_data;
 526         if (likely(new_data >= *offset))
 527                 *stat = (u32)(new_data - *offset);
 528         else
 529                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 530 }
 531
 532 /**
 533  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 534  * @vsi: the VSI to be updated
 535  **/
 536 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 537 {
 538         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 539         struct i40e_pf *pf = vsi->back;
 540         struct i40e_hw *hw = &pf->hw;
 541         struct i40e_eth_stats *oes;
 542         struct i40e_eth_stats *es;     /* device's eth stats */
 543
 544         es = &vsi->eth_stats;
 545         oes = &vsi->eth_stats_offsets;
 546
 547         /* Gather up the stats that the hw collects */
 548         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 549                            vsi->stat_offsets_loaded,
 550                            &oes->tx_errors, &es->tx_errors);
 551         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 552                            vsi->stat_offsets_loaded,
 553                            &oes->rx_discards, &es->rx_discards);
 554         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 555                            vsi->stat_offsets_loaded,
 556                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 557         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 558                            vsi->stat_offsets_loaded,
 559                            &oes->tx_errors, &es->tx_errors);
 560
 561         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 562                            I40E_GLV_GORCL(stat_idx),
 563                            vsi->stat_offsets_loaded,
 564                            &oes->rx_bytes, &es->rx_bytes);
 565         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 566                            I40E_GLV_UPRCL(stat_idx),
 567                            vsi->stat_offsets_loaded,
 568                            &oes->rx_unicast, &es->rx_unicast);
 569         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 570                            I40E_GLV_MPRCL(stat_idx),
 571                            vsi->stat_offsets_loaded,
 572                            &oes->rx_multicast, &es->rx_multicast);
 573         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 574                            I40E_GLV_BPRCL(stat_idx),
 575                            vsi->stat_offsets_loaded,
 576                            &oes->rx_broadcast, &es->rx_broadcast);
 577
 578         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 579                            I40E_GLV_GOTCL(stat_idx),
 580                            vsi->stat_offsets_loaded,
 581                            &oes->tx_bytes, &es->tx_bytes);
 582         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 583                            I40E_GLV_UPTCL(stat_idx),
 584                            vsi->stat_offsets_loaded,
 585                            &oes->tx_unicast, &es->tx_unicast);
 586         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 587                            I40E_GLV_MPTCL(stat_idx),
 588                            vsi->stat_offsets_loaded,
 589                            &oes->tx_multicast, &es->tx_multicast);
 590         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 591                            I40E_GLV_BPTCL(stat_idx),
 592                            vsi->stat_offsets_loaded,
 593                            &oes->tx_broadcast, &es->tx_broadcast);
 594         vsi->stat_offsets_loaded = true;
 595 }
 596
 597 /**
 598  * i40e_update_veb_stats - Update Switch component statistics
 599  * @veb: the VEB being updated
 600  **/
 601 static void i40e_update_veb_stats(struct i40e_veb *veb)
 602 {
 603         struct i40e_pf *pf = veb->pf;
 604         struct i40e_hw *hw = &pf->hw;
 605         struct i40e_eth_stats *oes;
 606         struct i40e_eth_stats *es;     /* device's eth stats */
 607         int idx = 0;
 608
 609         idx = veb->stats_idx;
 610         es = &veb->stats;
 611         oes = &veb->stats_offsets;
 612
 613         /* Gather up the stats that the hw collects */
 614         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 615                            veb->stat_offsets_loaded,
 616                            &oes->tx_discards, &es->tx_discards);
 617         if (hw->revision_id > 0)
 618                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 619                                    veb->stat_offsets_loaded,
 620                                    &oes->rx_unknown_protocol,
 621                                    &es->rx_unknown_protocol);
 622         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 623                            veb->stat_offsets_loaded,
 624                            &oes->rx_bytes, &es->rx_bytes);
 625         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 626                            veb->stat_offsets_loaded,
 627                            &oes->rx_unicast, &es->rx_unicast);
 628         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 629                            veb->stat_offsets_loaded,
 630                            &oes->rx_multicast, &es->rx_multicast);
 631         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 632                            veb->stat_offsets_loaded,
 633                            &oes->rx_broadcast, &es->rx_broadcast);
 634
 635         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 636                            veb->stat_offsets_loaded,
 637                            &oes->tx_bytes, &es->tx_bytes);
 638         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 639                            veb->stat_offsets_loaded,
 640                            &oes->tx_unicast, &es->tx_unicast);
 641         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 642                            veb->stat_offsets_loaded,
 643                            &oes->tx_multicast, &es->tx_multicast);
 644         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 645                            veb->stat_offsets_loaded,
 646                            &oes->tx_broadcast, &es->tx_broadcast);
 647         veb->stat_offsets_loaded = true;
 648 }
 649
 650 #ifdef I40E_FCOE
 651 /**
 652  * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters.
 653  * @vsi: the VSI that is capable of doing FCoE
 654  **/
 655 static void i40e_update_fcoe_stats(struct i40e_vsi *vsi)
 656 {
 657         struct i40e_pf *pf = vsi->back;
 658         struct i40e_hw *hw = &pf->hw;
 659         struct i40e_fcoe_stats *ofs;
 660         struct i40e_fcoe_stats *fs;     /* device's eth stats */
 661         int idx;
 662
 663         if (vsi->type != I40E_VSI_FCOE)
 664                 return;
 665
 666         idx = (pf->pf_seid - I40E_BASE_PF_SEID) + I40E_FCOE_PF_STAT_OFFSET;
 667         fs = &vsi->fcoe_stats;
 668         ofs = &vsi->fcoe_stats_offsets;
 669
 670         i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx),
 671                            vsi->fcoe_stat_offsets_loaded,
 672                            &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets);
 673         i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx),
 674                            vsi->fcoe_stat_offsets_loaded,
 675                            &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords);
 676         i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx),
 677                            vsi->fcoe_stat_offsets_loaded,
 678                            &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped);
 679         i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx),
 680                            vsi->fcoe_stat_offsets_loaded,
 681                            &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets);
 682         i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx),
 683                            vsi->fcoe_stat_offsets_loaded,
 684                            &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords);
 685         i40e_stat_update32(hw, I40E_GL_FCOECRC(idx),
 686                            vsi->fcoe_stat_offsets_loaded,
 687                            &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc);
 688         i40e_stat_update32(hw, I40E_GL_FCOELAST(idx),
 689                            vsi->fcoe_stat_offsets_loaded,
 690                            &ofs->fcoe_last_error, &fs->fcoe_last_error);
 691         i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx),
 692                            vsi->fcoe_stat_offsets_loaded,
 693                            &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count);
 694
 695         vsi->fcoe_stat_offsets_loaded = true;
 696 }
 697
 698 #endif
 699 /**
 700  * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 701  * @pf: the corresponding PF
 702  *
 703  * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 704  **/
 705 static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
 706 {
 707         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 708         struct i40e_hw_port_stats *nsd = &pf->stats;
 709         struct i40e_hw *hw = &pf->hw;
 710         u64 xoff = 0;
 711         u16 i, v;
 712
 713         if ((hw->fc.current_mode != I40E_FC_FULL) &&
 714             (hw->fc.current_mode != I40E_FC_RX_PAUSE))
 715                 return;
 716
 717         xoff = nsd->link_xoff_rx;
 718         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 719                            pf->stat_offsets_loaded,
 720                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 721
 722         /* No new LFC xoff rx */
 723         if (!(nsd->link_xoff_rx - xoff))
 724                 return;
 725
 726         /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
 727         for (v = 0; v < pf->num_alloc_vsi; v++) {
 728                 struct i40e_vsi *vsi = pf->vsi[v];
 729
 730                 if (!vsi || !vsi->tx_rings[0])
 731                         continue;
 732
 733                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 734                         struct i40e_ring *ring = vsi->tx_rings[i];
 735                         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
 736                 }
 737         }
 738 }
 739
 740 /**
 741  * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 742  * @pf: the corresponding PF
 743  *
 744  * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 745  **/
 746 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
 747 {
 748         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 749         struct i40e_hw_port_stats *nsd = &pf->stats;
 750         bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
 751         struct i40e_dcbx_config *dcb_cfg;
 752         struct i40e_hw *hw = &pf->hw;
 753         u16 i, v;
 754         u8 tc;
 755
 756         dcb_cfg = &hw->local_dcbx_config;
 757
 758         /* See if DCB enabled with PFC TC */
 759         if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
 760             !(dcb_cfg->pfc.pfcenable)) {
 761                 i40e_update_link_xoff_rx(pf);
 762                 return;
 763         }
 764
 765         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 766                 u64 prio_xoff = nsd->priority_xoff_rx[i];
 767                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 768                                    pf->stat_offsets_loaded,
 769                                    &osd->priority_xoff_rx[i],
 770                                    &nsd->priority_xoff_rx[i]);
 771
 772                 /* No new PFC xoff rx */
 773                 if (!(nsd->priority_xoff_rx[i] - prio_xoff))
 774                         continue;
 775                 /* Get the TC for given priority */
 776                 tc = dcb_cfg->etscfg.prioritytable[i];
 777                 xoff[tc] = true;
 778         }
 779
 780         /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
 781         for (v = 0; v < pf->num_alloc_vsi; v++) {
 782                 struct i40e_vsi *vsi = pf->vsi[v];
 783
 784                 if (!vsi || !vsi->tx_rings[0])
 785                         continue;
 786
 787                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 788                         struct i40e_ring *ring = vsi->tx_rings[i];
 789
 790                         tc = ring->dcb_tc;
 791                         if (xoff[tc])
 792                                 clear_bit(__I40E_HANG_CHECK_ARMED,
 793                                           &ring->state);
 794                 }
 795         }
 796 }
 797
 798 /**
 799  * i40e_update_vsi_stats - Update the vsi statistics counters.
 800  * @vsi: the VSI to be updated
 801  *
 802  * There are a few instances where we store the same stat in a
 803  * couple of different structs.  This is partly because we have
 804  * the netdev stats that need to be filled out, which is slightly
 805  * different from the "eth_stats" defined by the chip and used in
 806  * VF communications.  We sort it out here.
 807  **/
 808 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 809 {
 810         struct i40e_pf *pf = vsi->back;
 811         struct rtnl_link_stats64 *ons;
 812         struct rtnl_link_stats64 *ns;   /* netdev stats */
 813         struct i40e_eth_stats *oes;
 814         struct i40e_eth_stats *es;     /* device's eth stats */
 815         u32 tx_restart, tx_busy;
 816         u32 rx_page, rx_buf;
 817         u64 rx_p, rx_b;
 818         u64 tx_p, tx_b;
 819         u16 q;
 820
 821         if (test_bit(__I40E_DOWN, &vsi->state) ||
 822             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 823                 return;
 824
 825         ns = i40e_get_vsi_stats_struct(vsi);
 826         ons = &vsi->net_stats_offsets;
 827         es = &vsi->eth_stats;
 828         oes = &vsi->eth_stats_offsets;
 829
 830         /* Gather up the netdev and vsi stats that the driver collects
 831          * on the fly during packet processing
 832          */
 833         rx_b = rx_p = 0;
 834         tx_b = tx_p = 0;
 835         tx_restart = tx_busy = 0;
 836         rx_page = 0;
 837         rx_buf = 0;
 838         rcu_read_lock();
 839         for (q = 0; q < vsi->num_queue_pairs; q++) {
 840                 struct i40e_ring *p;
 841                 u64 bytes, packets;
 842                 unsigned int start;
 843
 844                 /* locate Tx ring */
 845                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 846
 847                 do {
 848                         start = u64_stats_fetch_begin_irq(&p->syncp);
 849                         packets = p->stats.packets;
 850                         bytes = p->stats.bytes;
 851                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 852                 tx_b += bytes;
 853                 tx_p += packets;
 854                 tx_restart += p->tx_stats.restart_queue;
 855                 tx_busy += p->tx_stats.tx_busy;
 856
 857                 /* Rx queue is part of the same block as Tx queue */
 858                 p = &p[1];
 859                 do {
 860                         start = u64_stats_fetch_begin_irq(&p->syncp);
 861                         packets = p->stats.packets;
 862                         bytes = p->stats.bytes;
 863                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 864                 rx_b += bytes;
 865                 rx_p += packets;
 866                 rx_buf += p->rx_stats.alloc_buff_failed;
 867                 rx_page += p->rx_stats.alloc_page_failed;
 868         }
 869         rcu_read_unlock();
 870         vsi->tx_restart = tx_restart;
 871         vsi->tx_busy = tx_busy;
 872         vsi->rx_page_failed = rx_page;
 873         vsi->rx_buf_failed = rx_buf;
 874
 875         ns->rx_packets = rx_p;
 876         ns->rx_bytes = rx_b;
 877         ns->tx_packets = tx_p;
 878         ns->tx_bytes = tx_b;
 879
 880         /* update netdev stats from eth stats */
 881         i40e_update_eth_stats(vsi);
 882         ons->tx_errors = oes->tx_errors;
 883         ns->tx_errors = es->tx_errors;
 884         ons->multicast = oes->rx_multicast;
 885         ns->multicast = es->rx_multicast;
 886         ons->rx_dropped = oes->rx_discards;
 887         ns->rx_dropped = es->rx_discards;
 888         ons->tx_dropped = oes->tx_discards;
 889         ns->tx_dropped = es->tx_discards;
 890
 891         /* pull in a couple PF stats if this is the main vsi */
 892         if (vsi == pf->vsi[pf->lan_vsi]) {
 893                 ns->rx_crc_errors = pf->stats.crc_errors;
 894                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 895                 ns->rx_length_errors = pf->stats.rx_length_errors;
 896         }
 897 }
 898
 899 /**
 900  * i40e_update_pf_stats - Update the pf statistics counters.
 901  * @pf: the PF to be updated
 902  **/
 903 static void i40e_update_pf_stats(struct i40e_pf *pf)
 904 {
 905         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 906         struct i40e_hw_port_stats *nsd = &pf->stats;
 907         struct i40e_hw *hw = &pf->hw;
 908         u32 val;
 909         int i;
 910
 911         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 912                            I40E_GLPRT_GORCL(hw->port),
 913                            pf->stat_offsets_loaded,
 914                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 915         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 916                            I40E_GLPRT_GOTCL(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 919         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 920                            pf->stat_offsets_loaded,
 921                            &osd->eth.rx_discards,
 922                            &nsd->eth.rx_discards);
 923         i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
 924                            pf->stat_offsets_loaded,
 925                            &osd->eth.tx_discards,
 926                            &nsd->eth.tx_discards);
 927
 928         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 929                            I40E_GLPRT_UPRCL(hw->port),
 930                            pf->stat_offsets_loaded,
 931                            &osd->eth.rx_unicast,
 932                            &nsd->eth.rx_unicast);
 933         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 934                            I40E_GLPRT_MPRCL(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->eth.rx_multicast,
 937                            &nsd->eth.rx_multicast);
 938         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 939                            I40E_GLPRT_BPRCL(hw->port),
 940                            pf->stat_offsets_loaded,
 941                            &osd->eth.rx_broadcast,
 942                            &nsd->eth.rx_broadcast);
 943         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 944                            I40E_GLPRT_UPTCL(hw->port),
 945                            pf->stat_offsets_loaded,
 946                            &osd->eth.tx_unicast,
 947                            &nsd->eth.tx_unicast);
 948         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 949                            I40E_GLPRT_MPTCL(hw->port),
 950                            pf->stat_offsets_loaded,
 951                            &osd->eth.tx_multicast,
 952                            &nsd->eth.tx_multicast);
 953         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 954                            I40E_GLPRT_BPTCL(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->eth.tx_broadcast,
 957                            &nsd->eth.tx_broadcast);
 958
 959         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 960                            pf->stat_offsets_loaded,
 961                            &osd->tx_dropped_link_down,
 962                            &nsd->tx_dropped_link_down);
 963
 964         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 965                            pf->stat_offsets_loaded,
 966                            &osd->crc_errors, &nsd->crc_errors);
 967
 968         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 969                            pf->stat_offsets_loaded,
 970                            &osd->illegal_bytes, &nsd->illegal_bytes);
 971
 972         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 973                            pf->stat_offsets_loaded,
 974                            &osd->mac_local_faults,
 975                            &nsd->mac_local_faults);
 976         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 977                            pf->stat_offsets_loaded,
 978                            &osd->mac_remote_faults,
 979                            &nsd->mac_remote_faults);
 980
 981         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 982                            pf->stat_offsets_loaded,
 983                            &osd->rx_length_errors,
 984                            &nsd->rx_length_errors);
 985
 986         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 987                            pf->stat_offsets_loaded,
 988                            &osd->link_xon_rx, &nsd->link_xon_rx);
 989         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 990                            pf->stat_offsets_loaded,
 991                            &osd->link_xon_tx, &nsd->link_xon_tx);
 992         i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
 993         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 994                            pf->stat_offsets_loaded,
 995                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 996
 997         for (i = 0; i < 8; i++) {
 998                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 999                                    pf->stat_offsets_loaded,
1000                                    &osd->priority_xon_rx[i],
1001                                    &nsd->priority_xon_rx[i]);
1002                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1003                                    pf->stat_offsets_loaded,
1004                                    &osd->priority_xon_tx[i],
1005                                    &nsd->priority_xon_tx[i]);
1006                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1007                                    pf->stat_offsets_loaded,
1008                                    &osd->priority_xoff_tx[i],
1009                                    &nsd->priority_xoff_tx[i]);
1010                 i40e_stat_update32(hw,
1011                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1012                                    pf->stat_offsets_loaded,
1013                                    &osd->priority_xon_2_xoff[i],
1014                                    &nsd->priority_xon_2_xoff[i]);
1015         }
1016
1017         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1018                            I40E_GLPRT_PRC64L(hw->port),
1019                            pf->stat_offsets_loaded,
1020                            &osd->rx_size_64, &nsd->rx_size_64);
1021         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1022                            I40E_GLPRT_PRC127L(hw->port),
1023                            pf->stat_offsets_loaded,
1024                            &osd->rx_size_127, &nsd->rx_size_127);
1025         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1026                            I40E_GLPRT_PRC255L(hw->port),
1027                            pf->stat_offsets_loaded,
1028                            &osd->rx_size_255, &nsd->rx_size_255);
1029         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1030                            I40E_GLPRT_PRC511L(hw->port),
1031                            pf->stat_offsets_loaded,
1032                            &osd->rx_size_511, &nsd->rx_size_511);
1033         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1034                            I40E_GLPRT_PRC1023L(hw->port),
1035                            pf->stat_offsets_loaded,
1036                            &osd->rx_size_1023, &nsd->rx_size_1023);
1037         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1038                            I40E_GLPRT_PRC1522L(hw->port),
1039                            pf->stat_offsets_loaded,
1040                            &osd->rx_size_1522, &nsd->rx_size_1522);
1041         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1042                            I40E_GLPRT_PRC9522L(hw->port),
1043                            pf->stat_offsets_loaded,
1044                            &osd->rx_size_big, &nsd->rx_size_big);
1045
1046         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1047                            I40E_GLPRT_PTC64L(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->tx_size_64, &nsd->tx_size_64);
1050         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1051                            I40E_GLPRT_PTC127L(hw->port),
1052                            pf->stat_offsets_loaded,
1053                            &osd->tx_size_127, &nsd->tx_size_127);
1054         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1055                            I40E_GLPRT_PTC255L(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->tx_size_255, &nsd->tx_size_255);
1058         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1059                            I40E_GLPRT_PTC511L(hw->port),
1060                            pf->stat_offsets_loaded,
1061                            &osd->tx_size_511, &nsd->tx_size_511);
1062         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1063                            I40E_GLPRT_PTC1023L(hw->port),
1064                            pf->stat_offsets_loaded,
1065                            &osd->tx_size_1023, &nsd->tx_size_1023);
1066         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1067                            I40E_GLPRT_PTC1522L(hw->port),
1068                            pf->stat_offsets_loaded,
1069                            &osd->tx_size_1522, &nsd->tx_size_1522);
1070         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1071                            I40E_GLPRT_PTC9522L(hw->port),
1072                            pf->stat_offsets_loaded,
1073                            &osd->tx_size_big, &nsd->tx_size_big);
1074
1075         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1076                            pf->stat_offsets_loaded,
1077                            &osd->rx_undersize, &nsd->rx_undersize);
1078         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1079                            pf->stat_offsets_loaded,
1080                            &osd->rx_fragments, &nsd->rx_fragments);
1081         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1082                            pf->stat_offsets_loaded,
1083                            &osd->rx_oversize, &nsd->rx_oversize);
1084         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1085                            pf->stat_offsets_loaded,
1086                            &osd->rx_jabber, &nsd->rx_jabber);
1087
1088         /* FDIR stats */
1089         i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_atr_cnt_idx),
1090                            pf->stat_offsets_loaded,
1091                            &osd->fd_atr_match, &nsd->fd_atr_match);
1092         i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_sb_cnt_idx),
1093                            pf->stat_offsets_loaded,
1094                            &osd->fd_sb_match, &nsd->fd_sb_match);
1095
1096         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1097         nsd->tx_lpi_status =
1098                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1099                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1100         nsd->rx_lpi_status =
1101                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1102                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1103         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1104                            pf->stat_offsets_loaded,
1105                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1106         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1107                            pf->stat_offsets_loaded,
1108                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1109
1110         pf->stat_offsets_loaded = true;
1111 }
1112
1113 /**
1114  * i40e_update_stats - Update the various statistics counters.
1115  * @vsi: the VSI to be updated
1116  *
1117  * Update the various stats for this VSI and its related entities.
1118  **/
1119 void i40e_update_stats(struct i40e_vsi *vsi)
1120 {
1121         struct i40e_pf *pf = vsi->back;
1122
1123         if (vsi == pf->vsi[pf->lan_vsi])
1124                 i40e_update_pf_stats(pf);
1125
1126         i40e_update_vsi_stats(vsi);
1127 #ifdef I40E_FCOE
1128         i40e_update_fcoe_stats(vsi);
1129 #endif
1130 }
1131
1132 /**
1133  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1134  * @vsi: the VSI to be searched
1135  * @macaddr: the MAC address
1136  * @vlan: the vlan
1137  * @is_vf: make sure its a vf filter, else doesn't matter
1138  * @is_netdev: make sure its a netdev filter, else doesn't matter
1139  *
1140  * Returns ptr to the filter object or NULL
1141  **/
1142 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1143                                                 u8 *macaddr, s16 vlan,
1144                                                 bool is_vf, bool is_netdev)
1145 {
1146         struct i40e_mac_filter *f;
1147
1148         if (!vsi || !macaddr)
1149                 return NULL;
1150
1151         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1152                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1153                     (vlan == f->vlan)    &&
1154                     (!is_vf || f->is_vf) &&
1155                     (!is_netdev || f->is_netdev))
1156                         return f;
1157         }
1158         return NULL;
1159 }
1160
1161 /**
1162  * i40e_find_mac - Find a mac addr in the macvlan filters list
1163  * @vsi: the VSI to be searched
1164  * @macaddr: the MAC address we are searching for
1165  * @is_vf: make sure its a vf filter, else doesn't matter
1166  * @is_netdev: make sure its a netdev filter, else doesn't matter
1167  *
1168  * Returns the first filter with the provided MAC address or NULL if
1169  * MAC address was not found
1170  **/
1171 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1172                                       bool is_vf, bool is_netdev)
1173 {
1174         struct i40e_mac_filter *f;
1175
1176         if (!vsi || !macaddr)
1177                 return NULL;
1178
1179         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1180                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1181                     (!is_vf || f->is_vf) &&
1182                     (!is_netdev || f->is_netdev))
1183                         return f;
1184         }
1185         return NULL;
1186 }
1187
1188 /**
1189  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1190  * @vsi: the VSI to be searched
1191  *
1192  * Returns true if VSI is in vlan mode or false otherwise
1193  **/
1194 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1195 {
1196         struct i40e_mac_filter *f;
1197
1198         /* Only -1 for all the filters denotes not in vlan mode
1199          * so we have to go through all the list in order to make sure
1200          */
1201         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1202                 if (f->vlan >= 0)
1203                         return true;
1204         }
1205
1206         return false;
1207 }
1208
1209 /**
1210  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1211  * @vsi: the VSI to be searched
1212  * @macaddr: the mac address to be filtered
1213  * @is_vf: true if it is a vf
1214  * @is_netdev: true if it is a netdev
1215  *
1216  * Goes through all the macvlan filters and adds a
1217  * macvlan filter for each unique vlan that already exists
1218  *
1219  * Returns first filter found on success, else NULL
1220  **/
1221 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1222                                              bool is_vf, bool is_netdev)
1223 {
1224         struct i40e_mac_filter *f;
1225
1226         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1227                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1228                                       is_vf, is_netdev)) {
1229                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1230                                              is_vf, is_netdev))
1231                                 return NULL;
1232                 }
1233         }
1234
1235         return list_first_entry_or_null(&vsi->mac_filter_list,
1236                                         struct i40e_mac_filter, list);
1237 }
1238
1239 /**
1240  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1241  * @vsi: the PF Main VSI - inappropriate for any other VSI
1242  * @macaddr: the MAC address
1243  *
1244  * Some older firmware configurations set up a default promiscuous VLAN
1245  * filter that needs to be removed.
1246  **/
1247 static int i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1248 {
1249         struct i40e_aqc_remove_macvlan_element_data element;
1250         struct i40e_pf *pf = vsi->back;
1251         i40e_status aq_ret;
1252
1253         /* Only appropriate for the PF main VSI */
1254         if (vsi->type != I40E_VSI_MAIN)
1255                 return -EINVAL;
1256
1257         memset(&element, 0, sizeof(element));
1258         ether_addr_copy(element.mac_addr, macaddr);
1259         element.vlan_tag = 0;
1260         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1261                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1262         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1263         if (aq_ret)
1264                 return -ENOENT;
1265
1266         return 0;
1267 }
1268
1269 /**
1270  * i40e_add_filter - Add a mac/vlan filter to the VSI
1271  * @vsi: the VSI to be searched
1272  * @macaddr: the MAC address
1273  * @vlan: the vlan
1274  * @is_vf: make sure its a vf filter, else doesn't matter
1275  * @is_netdev: make sure its a netdev filter, else doesn't matter
1276  *
1277  * Returns ptr to the filter object or NULL when no memory available.
1278  **/
1279 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1280                                         u8 *macaddr, s16 vlan,
1281                                         bool is_vf, bool is_netdev)
1282 {
1283         struct i40e_mac_filter *f;
1284
1285         if (!vsi || !macaddr)
1286                 return NULL;
1287
1288         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1289         if (!f) {
1290                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1291                 if (!f)
1292                         goto add_filter_out;
1293
1294                 ether_addr_copy(f->macaddr, macaddr);
1295                 f->vlan = vlan;
1296                 f->changed = true;
1297
1298                 INIT_LIST_HEAD(&f->list);
1299                 list_add(&f->list, &vsi->mac_filter_list);
1300         }
1301
1302         /* increment counter and add a new flag if needed */
1303         if (is_vf) {
1304                 if (!f->is_vf) {
1305                         f->is_vf = true;
1306                         f->counter++;
1307                 }
1308         } else if (is_netdev) {
1309                 if (!f->is_netdev) {
1310                         f->is_netdev = true;
1311                         f->counter++;
1312                 }
1313         } else {
1314                 f->counter++;
1315         }
1316
1317         /* changed tells sync_filters_subtask to
1318          * push the filter down to the firmware
1319          */
1320         if (f->changed) {
1321                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1322                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1323         }
1324
1325 add_filter_out:
1326         return f;
1327 }
1328
1329 /**
1330  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1331  * @vsi: the VSI to be searched
1332  * @macaddr: the MAC address
1333  * @vlan: the vlan
1334  * @is_vf: make sure it's a vf filter, else doesn't matter
1335  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1336  **/
1337 void i40e_del_filter(struct i40e_vsi *vsi,
1338                      u8 *macaddr, s16 vlan,
1339                      bool is_vf, bool is_netdev)
1340 {
1341         struct i40e_mac_filter *f;
1342
1343         if (!vsi || !macaddr)
1344                 return;
1345
1346         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1347         if (!f || f->counter == 0)
1348                 return;
1349
1350         if (is_vf) {
1351                 if (f->is_vf) {
1352                         f->is_vf = false;
1353                         f->counter--;
1354                 }
1355         } else if (is_netdev) {
1356                 if (f->is_netdev) {
1357                         f->is_netdev = false;
1358                         f->counter--;
1359                 }
1360         } else {
1361                 /* make sure we don't remove a filter in use by vf or netdev */
1362                 int min_f = 0;
1363                 min_f += (f->is_vf ? 1 : 0);
1364                 min_f += (f->is_netdev ? 1 : 0);
1365
1366                 if (f->counter > min_f)
1367                         f->counter--;
1368         }
1369
1370         /* counter == 0 tells sync_filters_subtask to
1371          * remove the filter from the firmware's list
1372          */
1373         if (f->counter == 0) {
1374                 f->changed = true;
1375                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1376                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1377         }
1378 }
1379
1380 /**
1381  * i40e_set_mac - NDO callback to set mac address
1382  * @netdev: network interface device structure
1383  * @p: pointer to an address structure
1384  *
1385  * Returns 0 on success, negative on failure
1386  **/
1387 #ifdef I40E_FCOE
1388 int i40e_set_mac(struct net_device *netdev, void *p)
1389 #else
1390 static int i40e_set_mac(struct net_device *netdev, void *p)
1391 #endif
1392 {
1393         struct i40e_netdev_priv *np = netdev_priv(netdev);
1394         struct i40e_vsi *vsi = np->vsi;
1395         struct i40e_pf *pf = vsi->back;
1396         struct i40e_hw *hw = &pf->hw;
1397         struct sockaddr *addr = p;
1398         struct i40e_mac_filter *f;
1399
1400         if (!is_valid_ether_addr(addr->sa_data))
1401                 return -EADDRNOTAVAIL;
1402
1403         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1404                 netdev_info(netdev, "already using mac address %pM\n",
1405                             addr->sa_data);
1406                 return 0;
1407         }
1408
1409         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1410             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1411                 return -EADDRNOTAVAIL;
1412
1413         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1414                 netdev_info(netdev, "returning to hw mac address %pM\n",
1415                             hw->mac.addr);
1416         else
1417                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1418
1419         if (vsi->type == I40E_VSI_MAIN) {
1420                 i40e_status ret;
1421                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1422                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1423                                                 addr->sa_data, NULL);
1424                 if (ret) {
1425                         netdev_info(netdev,
1426                                     "Addr change for Main VSI failed: %d\n",
1427                                     ret);
1428                         return -EADDRNOTAVAIL;
1429                 }
1430         }
1431
1432         if (ether_addr_equal(netdev->dev_addr, hw->mac.addr)) {
1433                 struct i40e_aqc_remove_macvlan_element_data element;
1434
1435                 memset(&element, 0, sizeof(element));
1436                 ether_addr_copy(element.mac_addr, netdev->dev_addr);
1437                 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1438                 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1439         } else {
1440                 i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1441                                 false, false);
1442         }
1443
1444         if (ether_addr_equal(addr->sa_data, hw->mac.addr)) {
1445                 struct i40e_aqc_add_macvlan_element_data element;
1446
1447                 memset(&element, 0, sizeof(element));
1448                 ether_addr_copy(element.mac_addr, hw->mac.addr);
1449                 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
1450                 i40e_aq_add_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1451         } else {
1452                 f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY,
1453                                     false, false);
1454                 if (f)
1455                         f->is_laa = true;
1456         }
1457
1458         i40e_sync_vsi_filters(vsi);
1459         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1460
1461         return 0;
1462 }
1463
1464 /**
1465  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1466  * @vsi: the VSI being setup
1467  * @ctxt: VSI context structure
1468  * @enabled_tc: Enabled TCs bitmap
1469  * @is_add: True if called before Add VSI
1470  *
1471  * Setup VSI queue mapping for enabled traffic classes.
1472  **/
1473 #ifdef I40E_FCOE
1474 void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1475                               struct i40e_vsi_context *ctxt,
1476                               u8 enabled_tc,
1477                               bool is_add)
1478 #else
1479 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1480                                      struct i40e_vsi_context *ctxt,
1481                                      u8 enabled_tc,
1482                                      bool is_add)
1483 #endif
1484 {
1485         struct i40e_pf *pf = vsi->back;
1486         u16 sections = 0;
1487         u8 netdev_tc = 0;
1488         u16 numtc = 0;
1489         u16 qcount;
1490         u8 offset;
1491         u16 qmap;
1492         int i;
1493         u16 num_tc_qps = 0;
1494
1495         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1496         offset = 0;
1497
1498         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1499                 /* Find numtc from enabled TC bitmap */
1500                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1501                         if (enabled_tc & (1 << i)) /* TC is enabled */
1502                                 numtc++;
1503                 }
1504                 if (!numtc) {
1505                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1506                         numtc = 1;
1507                 }
1508         } else {
1509                 /* At least TC0 is enabled in case of non-DCB case */
1510                 numtc = 1;
1511         }
1512
1513         vsi->tc_config.numtc = numtc;
1514         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1515         /* Number of queues per enabled TC */
1516         num_tc_qps = vsi->alloc_queue_pairs/numtc;
1517         num_tc_qps = min_t(int, num_tc_qps, I40E_MAX_QUEUES_PER_TC);
1518
1519         /* Setup queue offset/count for all TCs for given VSI */
1520         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1521                 /* See if the given TC is enabled for the given VSI */
1522                 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
1523                         int pow, num_qps;
1524
1525                         switch (vsi->type) {
1526                         case I40E_VSI_MAIN:
1527                                 qcount = min_t(int, pf->rss_size, num_tc_qps);
1528                                 break;
1529 #ifdef I40E_FCOE
1530                         case I40E_VSI_FCOE:
1531                                 qcount = num_tc_qps;
1532                                 break;
1533 #endif
1534                         case I40E_VSI_FDIR:
1535                         case I40E_VSI_SRIOV:
1536                         case I40E_VSI_VMDQ2:
1537                         default:
1538                                 qcount = num_tc_qps;
1539                                 WARN_ON(i != 0);
1540                                 break;
1541                         }
1542                         vsi->tc_config.tc_info[i].qoffset = offset;
1543                         vsi->tc_config.tc_info[i].qcount = qcount;
1544
1545                         /* find the power-of-2 of the number of queue pairs */
1546                         num_qps = qcount;
1547                         pow = 0;
1548                         while (num_qps && ((1 << pow) < qcount)) {
1549                                 pow++;
1550                                 num_qps >>= 1;
1551                         }
1552
1553                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1554                         qmap =
1555                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1556                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1557
1558                         offset += qcount;
1559                 } else {
1560                         /* TC is not enabled so set the offset to
1561                          * default queue and allocate one queue
1562                          * for the given TC.
1563                          */
1564                         vsi->tc_config.tc_info[i].qoffset = 0;
1565                         vsi->tc_config.tc_info[i].qcount = 1;
1566                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1567
1568                         qmap = 0;
1569                 }
1570                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1571         }
1572
1573         /* Set actual Tx/Rx queue pairs */
1574         vsi->num_queue_pairs = offset;
1575
1576         /* Scheduler section valid can only be set for ADD VSI */
1577         if (is_add) {
1578                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1579
1580                 ctxt->info.up_enable_bits = enabled_tc;
1581         }
1582         if (vsi->type == I40E_VSI_SRIOV) {
1583                 ctxt->info.mapping_flags |=
1584                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1585                 for (i = 0; i < vsi->num_queue_pairs; i++)
1586                         ctxt->info.queue_mapping[i] =
1587                                                cpu_to_le16(vsi->base_queue + i);
1588         } else {
1589                 ctxt->info.mapping_flags |=
1590                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1591                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1592         }
1593         ctxt->info.valid_sections |= cpu_to_le16(sections);
1594 }
1595
1596 /**
1597  * i40e_set_rx_mode - NDO callback to set the netdev filters
1598  * @netdev: network interface device structure
1599  **/
1600 #ifdef I40E_FCOE
1601 void i40e_set_rx_mode(struct net_device *netdev)
1602 #else
1603 static void i40e_set_rx_mode(struct net_device *netdev)
1604 #endif
1605 {
1606         struct i40e_netdev_priv *np = netdev_priv(netdev);
1607         struct i40e_mac_filter *f, *ftmp;
1608         struct i40e_vsi *vsi = np->vsi;
1609         struct netdev_hw_addr *uca;
1610         struct netdev_hw_addr *mca;
1611         struct netdev_hw_addr *ha;
1612
1613         /* add addr if not already in the filter list */
1614         netdev_for_each_uc_addr(uca, netdev) {
1615                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1616                         if (i40e_is_vsi_in_vlan(vsi))
1617                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1618                                                      false, true);
1619                         else
1620                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1621                                                 false, true);
1622                 }
1623         }
1624
1625         netdev_for_each_mc_addr(mca, netdev) {
1626                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1627                         if (i40e_is_vsi_in_vlan(vsi))
1628                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1629                                                      false, true);
1630                         else
1631                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1632                                                 false, true);
1633                 }
1634         }
1635
1636         /* remove filter if not in netdev list */
1637         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1638                 bool found = false;
1639
1640                 if (!f->is_netdev)
1641                         continue;
1642
1643                 if (is_multicast_ether_addr(f->macaddr)) {
1644                         netdev_for_each_mc_addr(mca, netdev) {
1645                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
1646                                         found = true;
1647                                         break;
1648                                 }
1649                         }
1650                 } else {
1651                         netdev_for_each_uc_addr(uca, netdev) {
1652                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
1653                                         found = true;
1654                                         break;
1655                                 }
1656                         }
1657
1658                         for_each_dev_addr(netdev, ha) {
1659                                 if (ether_addr_equal(ha->addr, f->macaddr)) {
1660                                         found = true;
1661                                         break;
1662                                 }
1663                         }
1664                 }
1665                 if (!found)
1666                         i40e_del_filter(
1667                            vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1668         }
1669
1670         /* check for other flag changes */
1671         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1672                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1673                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1674         }
1675 }
1676
1677 /**
1678  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1679  * @vsi: ptr to the VSI
1680  *
1681  * Push any outstanding VSI filter changes through the AdminQ.
1682  *
1683  * Returns 0 or error value
1684  **/
1685 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1686 {
1687         struct i40e_mac_filter *f, *ftmp;
1688         bool promisc_forced_on = false;
1689         bool add_happened = false;
1690         int filter_list_len = 0;
1691         u32 changed_flags = 0;
1692         i40e_status aq_ret = 0;
1693         struct i40e_pf *pf;
1694         int num_add = 0;
1695         int num_del = 0;
1696         u16 cmd_flags;
1697
1698         /* empty array typed pointers, kcalloc later */
1699         struct i40e_aqc_add_macvlan_element_data *add_list;
1700         struct i40e_aqc_remove_macvlan_element_data *del_list;
1701
1702         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1703                 usleep_range(1000, 2000);
1704         pf = vsi->back;
1705
1706         if (vsi->netdev) {
1707                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1708                 vsi->current_netdev_flags = vsi->netdev->flags;
1709         }
1710
1711         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1712                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1713
1714                 filter_list_len = pf->hw.aq.asq_buf_size /
1715                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1716                 del_list = kcalloc(filter_list_len,
1717                             sizeof(struct i40e_aqc_remove_macvlan_element_data),
1718                             GFP_KERNEL);
1719                 if (!del_list)
1720                         return -ENOMEM;
1721
1722                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1723                         if (!f->changed)
1724                                 continue;
1725
1726                         if (f->counter != 0)
1727                                 continue;
1728                         f->changed = false;
1729                         cmd_flags = 0;
1730
1731                         /* add to delete list */
1732                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
1733                         del_list[num_del].vlan_tag =
1734                                 cpu_to_le16((u16)(f->vlan ==
1735                                             I40E_VLAN_ANY ? 0 : f->vlan));
1736
1737                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1738                         del_list[num_del].flags = cmd_flags;
1739                         num_del++;
1740
1741                         /* unlink from filter list */
1742                         list_del(&f->list);
1743                         kfree(f);
1744
1745                         /* flush a full buffer */
1746                         if (num_del == filter_list_len) {
1747                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1748                                             vsi->seid, del_list, num_del,
1749                                             NULL);
1750                                 num_del = 0;
1751                                 memset(del_list, 0, sizeof(*del_list));
1752
1753                                 if (aq_ret &&
1754                                     pf->hw.aq.asq_last_status !=
1755                                                               I40E_AQ_RC_ENOENT)
1756                                         dev_info(&pf->pdev->dev,
1757                                                  "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
1758                                                  aq_ret,
1759                                                  pf->hw.aq.asq_last_status);
1760                         }
1761                 }
1762                 if (num_del) {
1763                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
1764                                                      del_list, num_del, NULL);
1765                         num_del = 0;
1766
1767                         if (aq_ret &&
1768                             pf->hw.aq.asq_last_status != I40E_AQ_RC_ENOENT)
1769                                 dev_info(&pf->pdev->dev,
1770                                          "ignoring delete macvlan error, err %d, aq_err %d\n",
1771                                          aq_ret, pf->hw.aq.asq_last_status);
1772                 }
1773
1774                 kfree(del_list);
1775                 del_list = NULL;
1776
1777                 /* do all the adds now */
1778                 filter_list_len = pf->hw.aq.asq_buf_size /
1779                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1780                 add_list = kcalloc(filter_list_len,
1781                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1782                                GFP_KERNEL);
1783                 if (!add_list)
1784                         return -ENOMEM;
1785
1786                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1787                         if (!f->changed)
1788                                 continue;
1789
1790                         if (f->counter == 0)
1791                                 continue;
1792                         f->changed = false;
1793                         add_happened = true;
1794                         cmd_flags = 0;
1795
1796                         /* add to add array */
1797                         ether_addr_copy(add_list[num_add].mac_addr, f->macaddr);
1798                         add_list[num_add].vlan_tag =
1799                                 cpu_to_le16(
1800                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
1801                         add_list[num_add].queue_number = 0;
1802
1803                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
1804                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
1805                         num_add++;
1806
1807                         /* flush a full buffer */
1808                         if (num_add == filter_list_len) {
1809                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1810                                                              add_list, num_add,
1811                                                              NULL);
1812                                 num_add = 0;
1813
1814                                 if (aq_ret)
1815                                         break;
1816                                 memset(add_list, 0, sizeof(*add_list));
1817                         }
1818                 }
1819                 if (num_add) {
1820                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1821                                                      add_list, num_add, NULL);
1822                         num_add = 0;
1823                 }
1824                 kfree(add_list);
1825                 add_list = NULL;
1826
1827                 if (add_happened && aq_ret &&
1828                     pf->hw.aq.asq_last_status != I40E_AQ_RC_EINVAL) {
1829                         dev_info(&pf->pdev->dev,
1830                                  "add filter failed, err %d, aq_err %d\n",
1831                                  aq_ret, pf->hw.aq.asq_last_status);
1832                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
1833                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1834                                       &vsi->state)) {
1835                                 promisc_forced_on = true;
1836                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1837                                         &vsi->state);
1838                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
1839                         }
1840                 }
1841         }
1842
1843         /* check for changes in promiscuous modes */
1844         if (changed_flags & IFF_ALLMULTI) {
1845                 bool cur_multipromisc;
1846                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
1847                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
1848                                                                vsi->seid,
1849                                                                cur_multipromisc,
1850                                                                NULL);
1851                 if (aq_ret)
1852                         dev_info(&pf->pdev->dev,
1853                                  "set multi promisc failed, err %d, aq_err %d\n",
1854                                  aq_ret, pf->hw.aq.asq_last_status);
1855         }
1856         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
1857                 bool cur_promisc;
1858                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
1859                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1860                                         &vsi->state));
1861                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
1862                                                              vsi->seid,
1863                                                              cur_promisc, NULL);
1864                 if (aq_ret)
1865                         dev_info(&pf->pdev->dev,
1866                                  "set uni promisc failed, err %d, aq_err %d\n",
1867                                  aq_ret, pf->hw.aq.asq_last_status);
1868                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
1869                                                    vsi->seid,
1870                                                    cur_promisc, NULL);
1871                 if (aq_ret)
1872                         dev_info(&pf->pdev->dev,
1873                                  "set brdcast promisc failed, err %d, aq_err %d\n",
1874                                  aq_ret, pf->hw.aq.asq_last_status);
1875         }
1876
1877         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
1878         return 0;
1879 }
1880
1881 /**
1882  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
1883  * @pf: board private structure
1884  **/
1885 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
1886 {
1887         int v;
1888
1889         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
1890                 return;
1891         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
1892
1893         for (v = 0; v < pf->num_alloc_vsi; v++) {
1894                 if (pf->vsi[v] &&
1895                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
1896                         i40e_sync_vsi_filters(pf->vsi[v]);
1897         }
1898 }
1899
1900 /**
1901  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
1902  * @netdev: network interface device structure
1903  * @new_mtu: new value for maximum frame size
1904  *
1905  * Returns 0 on success, negative on failure
1906  **/
1907 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
1908 {
1909         struct i40e_netdev_priv *np = netdev_priv(netdev);
1910         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
1911         struct i40e_vsi *vsi = np->vsi;
1912
1913         /* MTU < 68 is an error and causes problems on some kernels */
1914         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1915                 return -EINVAL;
1916
1917         netdev_info(netdev, "changing MTU from %d to %d\n",
1918                     netdev->mtu, new_mtu);
1919         netdev->mtu = new_mtu;
1920         if (netif_running(netdev))
1921                 i40e_vsi_reinit_locked(vsi);
1922
1923         return 0;
1924 }
1925
1926 /**
1927  * i40e_ioctl - Access the hwtstamp interface
1928  * @netdev: network interface device structure
1929  * @ifr: interface request data
1930  * @cmd: ioctl command
1931  **/
1932 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1933 {
1934         struct i40e_netdev_priv *np = netdev_priv(netdev);
1935         struct i40e_pf *pf = np->vsi->back;
1936
1937         switch (cmd) {
1938         case SIOCGHWTSTAMP:
1939                 return i40e_ptp_get_ts_config(pf, ifr);
1940         case SIOCSHWTSTAMP:
1941                 return i40e_ptp_set_ts_config(pf, ifr);
1942         default:
1943                 return -EOPNOTSUPP;
1944         }
1945 }
1946
1947 /**
1948  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
1949  * @vsi: the vsi being adjusted
1950  **/
1951 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
1952 {
1953         struct i40e_vsi_context ctxt;
1954         i40e_status ret;
1955
1956         if ((vsi->info.valid_sections &
1957              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1958             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
1959                 return;  /* already enabled */
1960
1961         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1962         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1963                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
1964
1965         ctxt.seid = vsi->seid;
1966         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1967         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1968         if (ret) {
1969                 dev_info(&vsi->back->pdev->dev,
1970                          "%s: update vsi failed, aq_err=%d\n",
1971                          __func__, vsi->back->hw.aq.asq_last_status);
1972         }
1973 }
1974
1975 /**
1976  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
1977  * @vsi: the vsi being adjusted
1978  **/
1979 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
1980 {
1981         struct i40e_vsi_context ctxt;
1982         i40e_status ret;
1983
1984         if ((vsi->info.valid_sections &
1985              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1986             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
1987              I40E_AQ_VSI_PVLAN_EMOD_MASK))
1988                 return;  /* already disabled */
1989
1990         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1991         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1992                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
1993
1994         ctxt.seid = vsi->seid;
1995         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1996         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1997         if (ret) {
1998                 dev_info(&vsi->back->pdev->dev,
1999                          "%s: update vsi failed, aq_err=%d\n",
2000                          __func__, vsi->back->hw.aq.asq_last_status);
2001         }
2002 }
2003
2004 /**
2005  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2006  * @netdev: network interface to be adjusted
2007  * @features: netdev features to test if VLAN offload is enabled or not
2008  **/
2009 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2010 {
2011         struct i40e_netdev_priv *np = netdev_priv(netdev);
2012         struct i40e_vsi *vsi = np->vsi;
2013
2014         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2015                 i40e_vlan_stripping_enable(vsi);
2016         else
2017                 i40e_vlan_stripping_disable(vsi);
2018 }
2019
2020 /**
2021  * i40e_vsi_add_vlan - Add vsi membership for given vlan
2022  * @vsi: the vsi being configured
2023  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2024  **/
2025 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
2026 {
2027         struct i40e_mac_filter *f, *add_f;
2028         bool is_netdev, is_vf;
2029
2030         is_vf = (vsi->type == I40E_VSI_SRIOV);
2031         is_netdev = !!(vsi->netdev);
2032
2033         if (is_netdev) {
2034                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
2035                                         is_vf, is_netdev);
2036                 if (!add_f) {
2037                         dev_info(&vsi->back->pdev->dev,
2038                                  "Could not add vlan filter %d for %pM\n",
2039                                  vid, vsi->netdev->dev_addr);
2040                         return -ENOMEM;
2041                 }
2042         }
2043
2044         list_for_each_entry(f, &vsi->mac_filter_list, list) {
2045                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
2046                 if (!add_f) {
2047                         dev_info(&vsi->back->pdev->dev,
2048                                  "Could not add vlan filter %d for %pM\n",
2049                                  vid, f->macaddr);
2050                         return -ENOMEM;
2051                 }
2052         }
2053
2054         /* Now if we add a vlan tag, make sure to check if it is the first
2055          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
2056          * with 0, so we now accept untagged and specified tagged traffic
2057          * (and not any taged and untagged)
2058          */
2059         if (vid > 0) {
2060                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
2061                                                   I40E_VLAN_ANY,
2062                                                   is_vf, is_netdev)) {
2063                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
2064                                         I40E_VLAN_ANY, is_vf, is_netdev);
2065                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
2066                                                 is_vf, is_netdev);
2067                         if (!add_f) {
2068                                 dev_info(&vsi->back->pdev->dev,
2069                                          "Could not add filter 0 for %pM\n",
2070                                          vsi->netdev->dev_addr);
2071                                 return -ENOMEM;
2072                         }
2073                 }
2074         }
2075
2076         /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */
2077         if (vid > 0 && !vsi->info.pvid) {
2078                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
2079                         if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2080                                              is_vf, is_netdev)) {
2081                                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2082                                                 is_vf, is_netdev);
2083                                 add_f = i40e_add_filter(vsi, f->macaddr,
2084                                                         0, is_vf, is_netdev);
2085                                 if (!add_f) {
2086                                         dev_info(&vsi->back->pdev->dev,
2087                                                  "Could not add filter 0 for %pM\n",
2088                                                  f->macaddr);
2089                                         return -ENOMEM;
2090                                 }
2091                         }
2092                 }
2093         }
2094
2095         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
2096             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
2097                 return 0;
2098
2099         return i40e_sync_vsi_filters(vsi);
2100 }
2101
2102 /**
2103  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
2104  * @vsi: the vsi being configured
2105  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2106  *
2107  * Return: 0 on success or negative otherwise
2108  **/
2109 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
2110 {
2111         struct net_device *netdev = vsi->netdev;
2112         struct i40e_mac_filter *f, *add_f;
2113         bool is_vf, is_netdev;
2114         int filter_count = 0;
2115
2116         is_vf = (vsi->type == I40E_VSI_SRIOV);
2117         is_netdev = !!(netdev);
2118
2119         if (is_netdev)
2120                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
2121
2122         list_for_each_entry(f, &vsi->mac_filter_list, list)
2123                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
2124
2125         /* go through all the filters for this VSI and if there is only
2126          * vid == 0 it means there are no other filters, so vid 0 must
2127          * be replaced with -1. This signifies that we should from now
2128          * on accept any traffic (with any tag present, or untagged)
2129          */
2130         list_for_each_entry(f, &vsi->mac_filter_list, list) {
2131                 if (is_netdev) {
2132                         if (f->vlan &&
2133                             ether_addr_equal(netdev->dev_addr, f->macaddr))
2134                                 filter_count++;
2135                 }
2136
2137                 if (f->vlan)
2138                         filter_count++;
2139         }
2140
2141         if (!filter_count && is_netdev) {
2142                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
2143                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
2144                                     is_vf, is_netdev);
2145                 if (!f) {
2146                         dev_info(&vsi->back->pdev->dev,
2147                                  "Could not add filter %d for %pM\n",
2148                                  I40E_VLAN_ANY, netdev->dev_addr);
2149                         return -ENOMEM;
2150                 }
2151         }
2152
2153         if (!filter_count) {
2154                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
2155                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
2156                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2157                                             is_vf, is_netdev);
2158                         if (!add_f) {
2159                                 dev_info(&vsi->back->pdev->dev,
2160                                          "Could not add filter %d for %pM\n",
2161                                          I40E_VLAN_ANY, f->macaddr);
2162                                 return -ENOMEM;
2163                         }
2164                 }
2165         }
2166
2167         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
2168             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
2169                 return 0;
2170
2171         return i40e_sync_vsi_filters(vsi);
2172 }
2173
2174 /**
2175  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2176  * @netdev: network interface to be adjusted
2177  * @vid: vlan id to be added
2178  *
2179  * net_device_ops implementation for adding vlan ids
2180  **/
2181 #ifdef I40E_FCOE
2182 int i40e_vlan_rx_add_vid(struct net_device *netdev,
2183                          __always_unused __be16 proto, u16 vid)
2184 #else
2185 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2186                                 __always_unused __be16 proto, u16 vid)
2187 #endif
2188 {
2189         struct i40e_netdev_priv *np = netdev_priv(netdev);
2190         struct i40e_vsi *vsi = np->vsi;
2191         int ret = 0;
2192
2193         if (vid > 4095)
2194                 return -EINVAL;
2195
2196         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
2197
2198         /* If the network stack called us with vid = 0 then
2199          * it is asking to receive priority tagged packets with
2200          * vlan id 0.  Our HW receives them by default when configured
2201          * to receive untagged packets so there is no need to add an
2202          * extra filter for vlan 0 tagged packets.
2203          */
2204         if (vid)
2205                 ret = i40e_vsi_add_vlan(vsi, vid);
2206
2207         if (!ret && (vid < VLAN_N_VID))
2208                 set_bit(vid, vsi->active_vlans);
2209
2210         return ret;
2211 }
2212
2213 /**
2214  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2215  * @netdev: network interface to be adjusted
2216  * @vid: vlan id to be removed
2217  *
2218  * net_device_ops implementation for removing vlan ids
2219  **/
2220 #ifdef I40E_FCOE
2221 int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2222                           __always_unused __be16 proto, u16 vid)
2223 #else
2224 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2225                                  __always_unused __be16 proto, u16 vid)
2226 #endif
2227 {
2228         struct i40e_netdev_priv *np = netdev_priv(netdev);
2229         struct i40e_vsi *vsi = np->vsi;
2230
2231         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
2232
2233         /* return code is ignored as there is nothing a user
2234          * can do about failure to remove and a log message was
2235          * already printed from the other function
2236          */
2237         i40e_vsi_kill_vlan(vsi, vid);
2238
2239         clear_bit(vid, vsi->active_vlans);
2240
2241         return 0;
2242 }
2243
2244 /**
2245  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2246  * @vsi: the vsi being brought back up
2247  **/
2248 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2249 {
2250         u16 vid;
2251
2252         if (!vsi->netdev)
2253                 return;
2254
2255         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2256
2257         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2258                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2259                                      vid);
2260 }
2261
2262 /**
2263  * i40e_vsi_add_pvid - Add pvid for the VSI
2264  * @vsi: the vsi being adjusted
2265  * @vid: the vlan id to set as a PVID
2266  **/
2267 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2268 {
2269         struct i40e_vsi_context ctxt;
2270         i40e_status aq_ret;
2271
2272         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2273         vsi->info.pvid = cpu_to_le16(vid);
2274         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2275                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2276                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2277
2278         ctxt.seid = vsi->seid;
2279         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2280         aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2281         if (aq_ret) {
2282                 dev_info(&vsi->back->pdev->dev,
2283                          "%s: update vsi failed, aq_err=%d\n",
2284                          __func__, vsi->back->hw.aq.asq_last_status);
2285                 return -ENOENT;
2286         }
2287
2288         return 0;
2289 }
2290
2291 /**
2292  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2293  * @vsi: the vsi being adjusted
2294  *
2295  * Just use the vlan_rx_register() service to put it back to normal
2296  **/
2297 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2298 {
2299         i40e_vlan_stripping_disable(vsi);
2300
2301         vsi->info.pvid = 0;
2302 }
2303
2304 /**
2305  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2306  * @vsi: ptr to the VSI
2307  *
2308  * If this function returns with an error, then it's possible one or
2309  * more of the rings is populated (while the rest are not).  It is the
2310  * callers duty to clean those orphaned rings.
2311  *
2312  * Return 0 on success, negative on failure
2313  **/
2314 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2315 {
2316         int i, err = 0;
2317
2318         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2319                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2320
2321         return err;
2322 }
2323
2324 /**
2325  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2326  * @vsi: ptr to the VSI
2327  *
2328  * Free VSI's transmit software resources
2329  **/
2330 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2331 {
2332         int i;
2333
2334         if (!vsi->tx_rings)
2335                 return;
2336
2337         for (i = 0; i < vsi->num_queue_pairs; i++)
2338                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2339                         i40e_free_tx_resources(vsi->tx_rings[i]);
2340 }
2341
2342 /**
2343  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2344  * @vsi: ptr to the VSI
2345  *
2346  * If this function returns with an error, then it's possible one or
2347  * more of the rings is populated (while the rest are not).  It is the
2348  * callers duty to clean those orphaned rings.
2349  *
2350  * Return 0 on success, negative on failure
2351  **/
2352 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2353 {
2354         int i, err = 0;
2355
2356         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2357                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2358 #ifdef I40E_FCOE
2359         i40e_fcoe_setup_ddp_resources(vsi);
2360 #endif
2361         return err;
2362 }
2363
2364 /**
2365  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2366  * @vsi: ptr to the VSI
2367  *
2368  * Free all receive software resources
2369  **/
2370 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2371 {
2372         int i;
2373
2374         if (!vsi->rx_rings)
2375                 return;
2376
2377         for (i = 0; i < vsi->num_queue_pairs; i++)
2378                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2379                         i40e_free_rx_resources(vsi->rx_rings[i]);
2380 #ifdef I40E_FCOE
2381         i40e_fcoe_free_ddp_resources(vsi);
2382 #endif
2383 }
2384
2385 /**
2386  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2387  * @ring: The Tx ring to configure
2388  *
2389  * Configure the Tx descriptor ring in the HMC context.
2390  **/
2391 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2392 {
2393         struct i40e_vsi *vsi = ring->vsi;
2394         u16 pf_q = vsi->base_queue + ring->queue_index;
2395         struct i40e_hw *hw = &vsi->back->hw;
2396         struct i40e_hmc_obj_txq tx_ctx;
2397         i40e_status err = 0;
2398         u32 qtx_ctl = 0;
2399
2400         /* some ATR related tx ring init */
2401         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2402                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2403                 ring->atr_count = 0;
2404         } else {
2405                 ring->atr_sample_rate = 0;
2406         }
2407
2408         /* initialize XPS */
2409         if (ring->q_vector && ring->netdev &&
2410             vsi->tc_config.numtc <= 1 &&
2411             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2412                 netif_set_xps_queue(ring->netdev,
2413                                     &ring->q_vector->affinity_mask,
2414                                     ring->queue_index);
2415
2416         /* clear the context structure first */
2417         memset(&tx_ctx, 0, sizeof(tx_ctx));
2418
2419         tx_ctx.new_context = 1;
2420         tx_ctx.base = (ring->dma / 128);
2421         tx_ctx.qlen = ring->count;
2422         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2423                                                I40E_FLAG_FD_ATR_ENABLED));
2424 #ifdef I40E_FCOE
2425         tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
2426 #endif
2427         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2428         /* FDIR VSI tx ring can still use RS bit and writebacks */
2429         if (vsi->type != I40E_VSI_FDIR)
2430                 tx_ctx.head_wb_ena = 1;
2431         tx_ctx.head_wb_addr = ring->dma +
2432                               (ring->count * sizeof(struct i40e_tx_desc));
2433
2434         /* As part of VSI creation/update, FW allocates certain
2435          * Tx arbitration queue sets for each TC enabled for
2436          * the VSI. The FW returns the handles to these queue
2437          * sets as part of the response buffer to Add VSI,
2438          * Update VSI, etc. AQ commands. It is expected that
2439          * these queue set handles be associated with the Tx
2440          * queues by the driver as part of the TX queue context
2441          * initialization. This has to be done regardless of
2442          * DCB as by default everything is mapped to TC0.
2443          */
2444         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2445         tx_ctx.rdylist_act = 0;
2446
2447         /* clear the context in the HMC */
2448         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2449         if (err) {
2450                 dev_info(&vsi->back->pdev->dev,
2451                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2452                          ring->queue_index, pf_q, err);
2453                 return -ENOMEM;
2454         }
2455
2456         /* set the context in the HMC */
2457         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2458         if (err) {
2459                 dev_info(&vsi->back->pdev->dev,
2460                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2461                          ring->queue_index, pf_q, err);
2462                 return -ENOMEM;
2463         }
2464
2465         /* Now associate this queue with this PCI function */
2466         if (vsi->type == I40E_VSI_VMDQ2)
2467                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2468         else
2469                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2470         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2471                     I40E_QTX_CTL_PF_INDX_MASK);
2472         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2473         i40e_flush(hw);
2474
2475         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
2476
2477         /* cache tail off for easier writes later */
2478         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2479
2480         return 0;
2481 }
2482
2483 /**
2484  * i40e_configure_rx_ring - Configure a receive ring context
2485  * @ring: The Rx ring to configure
2486  *
2487  * Configure the Rx descriptor ring in the HMC context.
2488  **/
2489 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2490 {
2491         struct i40e_vsi *vsi = ring->vsi;
2492         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2493         u16 pf_q = vsi->base_queue + ring->queue_index;
2494         struct i40e_hw *hw = &vsi->back->hw;
2495         struct i40e_hmc_obj_rxq rx_ctx;
2496         i40e_status err = 0;
2497
2498         ring->state = 0;
2499
2500         /* clear the context structure first */
2501         memset(&rx_ctx, 0, sizeof(rx_ctx));
2502
2503         ring->rx_buf_len = vsi->rx_buf_len;
2504         ring->rx_hdr_len = vsi->rx_hdr_len;
2505
2506         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2507         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2508
2509         rx_ctx.base = (ring->dma / 128);
2510         rx_ctx.qlen = ring->count;
2511
2512         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2513                 set_ring_16byte_desc_enabled(ring);
2514                 rx_ctx.dsize = 0;
2515         } else {
2516                 rx_ctx.dsize = 1;
2517         }
2518
2519         rx_ctx.dtype = vsi->dtype;
2520         if (vsi->dtype) {
2521                 set_ring_ps_enabled(ring);
2522                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2523                                   I40E_RX_SPLIT_IP      |
2524                                   I40E_RX_SPLIT_TCP_UDP |
2525                                   I40E_RX_SPLIT_SCTP;
2526         } else {
2527                 rx_ctx.hsplit_0 = 0;
2528         }
2529
2530         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2531                                   (chain_len * ring->rx_buf_len));
2532         if (hw->revision_id == 0)
2533                 rx_ctx.lrxqthresh = 0;
2534         else
2535                 rx_ctx.lrxqthresh = 2;
2536         rx_ctx.crcstrip = 1;
2537         rx_ctx.l2tsel = 1;
2538         rx_ctx.showiv = 1;
2539 #ifdef I40E_FCOE
2540         rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
2541 #endif
2542         /* set the prefena field to 1 because the manual says to */
2543         rx_ctx.prefena = 1;
2544
2545         /* clear the context in the HMC */
2546         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2547         if (err) {
2548                 dev_info(&vsi->back->pdev->dev,
2549                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2550                          ring->queue_index, pf_q, err);
2551                 return -ENOMEM;
2552         }
2553
2554         /* set the context in the HMC */
2555         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2556         if (err) {
2557                 dev_info(&vsi->back->pdev->dev,
2558                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2559                          ring->queue_index, pf_q, err);
2560                 return -ENOMEM;
2561         }
2562
2563         /* cache tail for quicker writes, and clear the reg before use */
2564         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2565         writel(0, ring->tail);
2566
2567         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
2568
2569         return 0;
2570 }
2571
2572 /**
2573  * i40e_vsi_configure_tx - Configure the VSI for Tx
2574  * @vsi: VSI structure describing this set of rings and resources
2575  *
2576  * Configure the Tx VSI for operation.
2577  **/
2578 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2579 {
2580         int err = 0;
2581         u16 i;
2582
2583         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2584                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2585
2586         return err;
2587 }
2588
2589 /**
2590  * i40e_vsi_configure_rx - Configure the VSI for Rx
2591  * @vsi: the VSI being configured
2592  *
2593  * Configure the Rx VSI for operation.
2594  **/
2595 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2596 {
2597         int err = 0;
2598         u16 i;
2599
2600         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2601                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2602                                + ETH_FCS_LEN + VLAN_HLEN;
2603         else
2604                 vsi->max_frame = I40E_RXBUFFER_2048;
2605
2606         /* figure out correct receive buffer length */
2607         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2608                                     I40E_FLAG_RX_PS_ENABLED)) {
2609         case I40E_FLAG_RX_1BUF_ENABLED:
2610                 vsi->rx_hdr_len = 0;
2611                 vsi->rx_buf_len = vsi->max_frame;
2612                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2613                 break;
2614         case I40E_FLAG_RX_PS_ENABLED:
2615                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2616                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2617                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2618                 break;
2619         default:
2620                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2621                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2622                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2623                 break;
2624         }
2625
2626 #ifdef I40E_FCOE
2627         /* setup rx buffer for FCoE */
2628         if ((vsi->type == I40E_VSI_FCOE) &&
2629             (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) {
2630                 vsi->rx_hdr_len = 0;
2631                 vsi->rx_buf_len = I40E_RXBUFFER_3072;
2632                 vsi->max_frame = I40E_RXBUFFER_3072;
2633                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2634         }
2635
2636 #endif /* I40E_FCOE */
2637         /* round up for the chip's needs */
2638         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
2639                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2640         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
2641                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2642
2643         /* set up individual rings */
2644         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2645                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
2646
2647         return err;
2648 }
2649
2650 /**
2651  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
2652  * @vsi: ptr to the VSI
2653  **/
2654 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
2655 {
2656         struct i40e_ring *tx_ring, *rx_ring;
2657         u16 qoffset, qcount;
2658         int i, n;
2659
2660         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2661                 return;
2662
2663         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
2664                 if (!(vsi->tc_config.enabled_tc & (1 << n)))
2665                         continue;
2666
2667                 qoffset = vsi->tc_config.tc_info[n].qoffset;
2668                 qcount = vsi->tc_config.tc_info[n].qcount;
2669                 for (i = qoffset; i < (qoffset + qcount); i++) {
2670                         rx_ring = vsi->rx_rings[i];
2671                         tx_ring = vsi->tx_rings[i];
2672                         rx_ring->dcb_tc = n;
2673                         tx_ring->dcb_tc = n;
2674                 }
2675         }
2676 }
2677
2678 /**
2679  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
2680  * @vsi: ptr to the VSI
2681  **/
2682 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
2683 {
2684         if (vsi->netdev)
2685                 i40e_set_rx_mode(vsi->netdev);
2686 }
2687
2688 /**
2689  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
2690  * @vsi: Pointer to the targeted VSI
2691  *
2692  * This function replays the hlist on the hw where all the SB Flow Director
2693  * filters were saved.
2694  **/
2695 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
2696 {
2697         struct i40e_fdir_filter *filter;
2698         struct i40e_pf *pf = vsi->back;
2699         struct hlist_node *node;
2700
2701         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
2702                 return;
2703
2704         hlist_for_each_entry_safe(filter, node,
2705                                   &pf->fdir_filter_list, fdir_node) {
2706                 i40e_add_del_fdir(vsi, filter, true);
2707         }
2708 }
2709
2710 /**
2711  * i40e_vsi_configure - Set up the VSI for action
2712  * @vsi: the VSI being configured
2713  **/
2714 static int i40e_vsi_configure(struct i40e_vsi *vsi)
2715 {
2716         int err;
2717
2718         i40e_set_vsi_rx_mode(vsi);
2719         i40e_restore_vlan(vsi);
2720         i40e_vsi_config_dcb_rings(vsi);
2721         err = i40e_vsi_configure_tx(vsi);
2722         if (!err)
2723                 err = i40e_vsi_configure_rx(vsi);
2724
2725         return err;
2726 }
2727
2728 /**
2729  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
2730  * @vsi: the VSI being configured
2731  **/
2732 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
2733 {
2734         struct i40e_pf *pf = vsi->back;
2735         struct i40e_q_vector *q_vector;
2736         struct i40e_hw *hw = &pf->hw;
2737         u16 vector;
2738         int i, q;
2739         u32 val;
2740         u32 qp;
2741
2742         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
2743          * and PFINT_LNKLSTn registers, e.g.:
2744          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
2745          */
2746         qp = vsi->base_queue;
2747         vector = vsi->base_vector;
2748         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
2749                 q_vector = vsi->q_vectors[i];
2750                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2751                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
2752                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
2753                      q_vector->rx.itr);
2754                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2755                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
2756                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
2757                      q_vector->tx.itr);
2758
2759                 /* Linked list for the queuepairs assigned to this vector */
2760                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
2761                 for (q = 0; q < q_vector->num_ringpairs; q++) {
2762                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2763                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
2764                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2765                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
2766                               (I40E_QUEUE_TYPE_TX
2767                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2768
2769                         wr32(hw, I40E_QINT_RQCTL(qp), val);
2770
2771                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2772                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
2773                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2774                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
2775                               (I40E_QUEUE_TYPE_RX
2776                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2777
2778                         /* Terminate the linked list */
2779                         if (q == (q_vector->num_ringpairs - 1))
2780                                 val |= (I40E_QUEUE_END_OF_LIST
2781                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2782
2783                         wr32(hw, I40E_QINT_TQCTL(qp), val);
2784                         qp++;
2785                 }
2786         }
2787
2788         i40e_flush(hw);
2789 }
2790
2791 /**
2792  * i40e_enable_misc_int_causes - enable the non-queue interrupts
2793  * @hw: ptr to the hardware info
2794  **/
2795 static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
2796 {
2797         u32 val;
2798
2799         /* clear things first */
2800         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2801         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2802
2803         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
2804               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
2805               I40E_PFINT_ICR0_ENA_GRST_MASK          |
2806               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
2807               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
2808               I40E_PFINT_ICR0_ENA_TIMESYNC_MASK      |
2809               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
2810               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
2811               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2812
2813         wr32(hw, I40E_PFINT_ICR0_ENA, val);
2814
2815         /* SW_ITR_IDX = 0, but don't change INTENA */
2816         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2817                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2818
2819         /* OTHER_ITR_IDX = 0 */
2820         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2821 }
2822
2823 /**
2824  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
2825  * @vsi: the VSI being configured
2826  **/
2827 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
2828 {
2829         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
2830         struct i40e_pf *pf = vsi->back;
2831         struct i40e_hw *hw = &pf->hw;
2832         u32 val;
2833
2834         /* set the ITR configuration */
2835         q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2836         q_vector->rx.latency_range = I40E_LOW_LATENCY;
2837         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
2838         q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2839         q_vector->tx.latency_range = I40E_LOW_LATENCY;
2840         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
2841
2842         i40e_enable_misc_int_causes(hw);
2843
2844         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2845         wr32(hw, I40E_PFINT_LNKLST0, 0);
2846
2847         /* Associate the queue pair to the vector and enable the queue int */
2848         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
2849               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2850               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2851
2852         wr32(hw, I40E_QINT_RQCTL(0), val);
2853
2854         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
2855               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2856               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2857
2858         wr32(hw, I40E_QINT_TQCTL(0), val);
2859         i40e_flush(hw);
2860 }
2861
2862 /**
2863  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
2864  * @pf: board private structure
2865  **/
2866 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
2867 {
2868         struct i40e_hw *hw = &pf->hw;
2869
2870         wr32(hw, I40E_PFINT_DYN_CTL0,
2871              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2872         i40e_flush(hw);
2873 }
2874
2875 /**
2876  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
2877  * @pf: board private structure
2878  **/
2879 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
2880 {
2881         struct i40e_hw *hw = &pf->hw;
2882         u32 val;
2883
2884         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
2885               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2886               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2887
2888         wr32(hw, I40E_PFINT_DYN_CTL0, val);
2889         i40e_flush(hw);
2890 }
2891
2892 /**
2893  * i40e_irq_dynamic_enable - Enable default interrupt generation settings
2894  * @vsi: pointer to a vsi
2895  * @vector: enable a particular Hw Interrupt vector
2896  **/
2897 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
2898 {
2899         struct i40e_pf *pf = vsi->back;
2900         struct i40e_hw *hw = &pf->hw;
2901         u32 val;
2902
2903         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2904               I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2905               (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2906         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2907         /* skip the flush */
2908 }
2909
2910 /**
2911  * i40e_irq_dynamic_disable - Disable default interrupt generation settings
2912  * @vsi: pointer to a vsi
2913  * @vector: enable a particular Hw Interrupt vector
2914  **/
2915 void i40e_irq_dynamic_disable(struct i40e_vsi *vsi, int vector)
2916 {
2917         struct i40e_pf *pf = vsi->back;
2918         struct i40e_hw *hw = &pf->hw;
2919         u32 val;
2920
2921         val = I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
2922         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2923         i40e_flush(hw);
2924 }
2925
2926 /**
2927  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
2928  * @irq: interrupt number
2929  * @data: pointer to a q_vector
2930  **/
2931 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
2932 {
2933         struct i40e_q_vector *q_vector = data;
2934
2935         if (!q_vector->tx.ring && !q_vector->rx.ring)
2936                 return IRQ_HANDLED;
2937
2938         napi_schedule(&q_vector->napi);
2939
2940         return IRQ_HANDLED;
2941 }
2942
2943 /**
2944  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
2945  * @vsi: the VSI being configured
2946  * @basename: name for the vector
2947  *
2948  * Allocates MSI-X vectors and requests interrupts from the kernel.
2949  **/
2950 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
2951 {
2952         int q_vectors = vsi->num_q_vectors;
2953         struct i40e_pf *pf = vsi->back;
2954         int base = vsi->base_vector;
2955         int rx_int_idx = 0;
2956         int tx_int_idx = 0;
2957         int vector, err;
2958
2959         for (vector = 0; vector < q_vectors; vector++) {
2960                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
2961
2962                 if (q_vector->tx.ring && q_vector->rx.ring) {
2963                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2964                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2965                         tx_int_idx++;
2966                 } else if (q_vector->rx.ring) {
2967                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2968                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
2969                 } else if (q_vector->tx.ring) {
2970                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2971                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
2972                 } else {
2973                         /* skip this unused q_vector */
2974                         continue;
2975                 }
2976                 err = request_irq(pf->msix_entries[base + vector].vector,
2977                                   vsi->irq_handler,
2978                                   0,
2979                                   q_vector->name,
2980                                   q_vector);
2981                 if (err) {
2982                         dev_info(&pf->pdev->dev,
2983                                  "%s: request_irq failed, error: %d\n",
2984                                  __func__, err);
2985                         goto free_queue_irqs;
2986                 }
2987                 /* assign the mask for this irq */
2988                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2989                                       &q_vector->affinity_mask);
2990         }
2991
2992         vsi->irqs_ready = true;
2993         return 0;
2994
2995 free_queue_irqs:
2996         while (vector) {
2997                 vector--;
2998                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2999                                       NULL);
3000                 free_irq(pf->msix_entries[base + vector].vector,
3001                          &(vsi->q_vectors[vector]));
3002         }
3003         return err;
3004 }
3005
3006 /**
3007  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3008  * @vsi: the VSI being un-configured
3009  **/
3010 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3011 {
3012         struct i40e_pf *pf = vsi->back;
3013         struct i40e_hw *hw = &pf->hw;
3014         int base = vsi->base_vector;
3015         int i;
3016
3017         for (i = 0; i < vsi->num_queue_pairs; i++) {
3018                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
3019                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
3020         }
3021
3022         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3023                 for (i = vsi->base_vector;
3024                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3025                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3026
3027                 i40e_flush(hw);
3028                 for (i = 0; i < vsi->num_q_vectors; i++)
3029                         synchronize_irq(pf->msix_entries[i + base].vector);
3030         } else {
3031                 /* Legacy and MSI mode - this stops all interrupt handling */
3032                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3033                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3034                 i40e_flush(hw);
3035                 synchronize_irq(pf->pdev->irq);
3036         }
3037 }
3038
3039 /**
3040  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3041  * @vsi: the VSI being configured
3042  **/
3043 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3044 {
3045         struct i40e_pf *pf = vsi->back;
3046         int i;
3047
3048         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3049                 for (i = vsi->base_vector;
3050                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3051                         i40e_irq_dynamic_enable(vsi, i);
3052         } else {
3053                 i40e_irq_dynamic_enable_icr0(pf);
3054         }
3055
3056         i40e_flush(&pf->hw);
3057         return 0;
3058 }
3059
3060 /**
3061  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3062  * @pf: board private structure
3063  **/
3064 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3065 {
3066         /* Disable ICR 0 */
3067         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3068         i40e_flush(&pf->hw);
3069 }
3070
3071 /**
3072  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3073  * @irq: interrupt number
3074  * @data: pointer to a q_vector
3075  *
3076  * This is the handler used for all MSI/Legacy interrupts, and deals
3077  * with both queue and non-queue interrupts.  This is also used in
3078  * MSIX mode to handle the non-queue interrupts.
3079  **/
3080 static irqreturn_t i40e_intr(int irq, void *data)
3081 {
3082         struct i40e_pf *pf = (struct i40e_pf *)data;
3083         struct i40e_hw *hw = &pf->hw;
3084         irqreturn_t ret = IRQ_NONE;
3085         u32 icr0, icr0_remaining;
3086         u32 val, ena_mask;
3087
3088         icr0 = rd32(hw, I40E_PFINT_ICR0);
3089         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3090
3091         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3092         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3093                 goto enable_intr;
3094
3095         /* if interrupt but no bits showing, must be SWINT */
3096         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3097             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3098                 pf->sw_int_count++;
3099
3100         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3101         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3102
3103                 /* temporarily disable queue cause for NAPI processing */
3104                 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
3105                 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3106                 wr32(hw, I40E_QINT_RQCTL(0), qval);
3107
3108                 qval = rd32(hw, I40E_QINT_TQCTL(0));
3109                 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3110                 wr32(hw, I40E_QINT_TQCTL(0), qval);
3111
3112                 if (!test_bit(__I40E_DOWN, &pf->state))
3113                         napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
3114         }
3115
3116         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3117                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3118                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
3119         }
3120
3121         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3122                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3123                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
3124         }
3125
3126         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3127                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3128                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
3129         }
3130
3131         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3132                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
3133                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
3134                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3135                 val = rd32(hw, I40E_GLGEN_RSTAT);
3136                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3137                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3138                 if (val == I40E_RESET_CORER) {
3139                         pf->corer_count++;
3140                 } else if (val == I40E_RESET_GLOBR) {
3141                         pf->globr_count++;
3142                 } else if (val == I40E_RESET_EMPR) {
3143                         pf->empr_count++;
3144                         set_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
3145                 }
3146         }
3147
3148         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3149                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3150                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3151         }
3152
3153         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3154                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3155
3156                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3157                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3158                         i40e_ptp_tx_hwtstamp(pf);
3159                 }
3160         }
3161
3162         /* If a critical error is pending we have no choice but to reset the
3163          * device.
3164          * Report and mask out any remaining unexpected interrupts.
3165          */
3166         icr0_remaining = icr0 & ena_mask;
3167         if (icr0_remaining) {
3168                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3169                          icr0_remaining);
3170                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3171                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3172                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3173                         dev_info(&pf->pdev->dev, "device will be reset\n");
3174                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
3175                         i40e_service_event_schedule(pf);
3176                 }
3177                 ena_mask &= ~icr0_remaining;
3178         }
3179         ret = IRQ_HANDLED;
3180
3181 enable_intr:
3182         /* re-enable interrupt causes */
3183         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3184         if (!test_bit(__I40E_DOWN, &pf->state)) {
3185                 i40e_service_event_schedule(pf);
3186                 i40e_irq_dynamic_enable_icr0(pf);
3187         }
3188
3189         return ret;
3190 }
3191
3192 /**
3193  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3194  * @tx_ring:  tx ring to clean
3195  * @budget:   how many cleans we're allowed
3196  *
3197  * Returns true if there's any budget left (e.g. the clean is finished)
3198  **/
3199 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3200 {
3201         struct i40e_vsi *vsi = tx_ring->vsi;
3202         u16 i = tx_ring->next_to_clean;
3203         struct i40e_tx_buffer *tx_buf;
3204         struct i40e_tx_desc *tx_desc;
3205
3206         tx_buf = &tx_ring->tx_bi[i];
3207         tx_desc = I40E_TX_DESC(tx_ring, i);
3208         i -= tx_ring->count;
3209
3210         do {
3211                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3212
3213                 /* if next_to_watch is not set then there is no work pending */
3214                 if (!eop_desc)
3215                         break;
3216
3217                 /* prevent any other reads prior to eop_desc */
3218                 read_barrier_depends();
3219
3220                 /* if the descriptor isn't done, no work yet to do */
3221                 if (!(eop_desc->cmd_type_offset_bsz &
3222                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3223                         break;
3224
3225                 /* clear next_to_watch to prevent false hangs */
3226                 tx_buf->next_to_watch = NULL;
3227
3228                 tx_desc->buffer_addr = 0;
3229                 tx_desc->cmd_type_offset_bsz = 0;
3230                 /* move past filter desc */
3231                 tx_buf++;
3232                 tx_desc++;
3233                 i++;
3234                 if (unlikely(!i)) {
3235                         i -= tx_ring->count;
3236                         tx_buf = tx_ring->tx_bi;
3237                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3238                 }
3239                 /* unmap skb header data */
3240                 dma_unmap_single(tx_ring->dev,
3241                                  dma_unmap_addr(tx_buf, dma),
3242                                  dma_unmap_len(tx_buf, len),
3243                                  DMA_TO_DEVICE);
3244                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3245                         kfree(tx_buf->raw_buf);
3246
3247                 tx_buf->raw_buf = NULL;
3248                 tx_buf->tx_flags = 0;
3249                 tx_buf->next_to_watch = NULL;
3250                 dma_unmap_len_set(tx_buf, len, 0);
3251                 tx_desc->buffer_addr = 0;
3252                 tx_desc->cmd_type_offset_bsz = 0;
3253
3254                 /* move us past the eop_desc for start of next FD desc */
3255                 tx_buf++;
3256                 tx_desc++;
3257                 i++;
3258                 if (unlikely(!i)) {
3259                         i -= tx_ring->count;
3260                         tx_buf = tx_ring->tx_bi;
3261                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3262                 }
3263
3264                 /* update budget accounting */
3265                 budget--;
3266         } while (likely(budget));
3267
3268         i += tx_ring->count;
3269         tx_ring->next_to_clean = i;
3270
3271         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
3272                 i40e_irq_dynamic_enable(vsi,
3273                                 tx_ring->q_vector->v_idx + vsi->base_vector);
3274         }
3275         return budget > 0;
3276 }
3277
3278 /**
3279  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3280  * @irq: interrupt number
3281  * @data: pointer to a q_vector
3282  **/
3283 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3284 {
3285         struct i40e_q_vector *q_vector = data;
3286         struct i40e_vsi *vsi;
3287
3288         if (!q_vector->tx.ring)
3289                 return IRQ_HANDLED;
3290
3291         vsi = q_vector->tx.ring->vsi;
3292         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3293
3294         return IRQ_HANDLED;
3295 }
3296
3297 /**
3298  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3299  * @vsi: the VSI being configured
3300  * @v_idx: vector index
3301  * @qp_idx: queue pair index
3302  **/
3303 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3304 {
3305         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3306         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3307         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3308
3309         tx_ring->q_vector = q_vector;
3310         tx_ring->next = q_vector->tx.ring;
3311         q_vector->tx.ring = tx_ring;
3312         q_vector->tx.count++;
3313
3314         rx_ring->q_vector = q_vector;
3315         rx_ring->next = q_vector->rx.ring;
3316         q_vector->rx.ring = rx_ring;
3317         q_vector->rx.count++;
3318 }
3319
3320 /**
3321  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3322  * @vsi: the VSI being configured
3323  *
3324  * This function maps descriptor rings to the queue-specific vectors
3325  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3326  * one vector per queue pair, but on a constrained vector budget, we
3327  * group the queue pairs as "efficiently" as possible.
3328  **/
3329 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3330 {
3331         int qp_remaining = vsi->num_queue_pairs;
3332         int q_vectors = vsi->num_q_vectors;
3333         int num_ringpairs;
3334         int v_start = 0;
3335         int qp_idx = 0;
3336
3337         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3338          * group them so there are multiple queues per vector.
3339          * It is also important to go through all the vectors available to be
3340          * sure that if we don't use all the vectors, that the remaining vectors
3341          * are cleared. This is especially important when decreasing the
3342          * number of queues in use.
3343          */
3344         for (; v_start < q_vectors; v_start++) {
3345                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3346
3347                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3348
3349                 q_vector->num_ringpairs = num_ringpairs;
3350
3351                 q_vector->rx.count = 0;
3352                 q_vector->tx.count = 0;
3353                 q_vector->rx.ring = NULL;
3354                 q_vector->tx.ring = NULL;
3355
3356                 while (num_ringpairs--) {
3357                         map_vector_to_qp(vsi, v_start, qp_idx);
3358                         qp_idx++;
3359                         qp_remaining--;
3360                 }
3361         }
3362 }
3363
3364 /**
3365  * i40e_vsi_request_irq - Request IRQ from the OS
3366  * @vsi: the VSI being configured
3367  * @basename: name for the vector
3368  **/
3369 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3370 {
3371         struct i40e_pf *pf = vsi->back;
3372         int err;
3373
3374         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3375                 err = i40e_vsi_request_irq_msix(vsi, basename);
3376         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3377                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3378                                   pf->misc_int_name, pf);
3379         else
3380                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3381                                   pf->misc_int_name, pf);
3382
3383         if (err)
3384                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3385
3386         return err;
3387 }
3388
3389 #ifdef CONFIG_NET_POLL_CONTROLLER
3390 /**
3391  * i40e_netpoll - A Polling 'interrupt'handler
3392  * @netdev: network interface device structure
3393  *
3394  * This is used by netconsole to send skbs without having to re-enable
3395  * interrupts.  It's not called while the normal interrupt routine is executing.
3396  **/
3397 #ifdef I40E_FCOE
3398 void i40e_netpoll(struct net_device *netdev)
3399 #else
3400 static void i40e_netpoll(struct net_device *netdev)
3401 #endif
3402 {
3403         struct i40e_netdev_priv *np = netdev_priv(netdev);
3404         struct i40e_vsi *vsi = np->vsi;
3405         struct i40e_pf *pf = vsi->back;
3406         int i;
3407
3408         /* if interface is down do nothing */
3409         if (test_bit(__I40E_DOWN, &vsi->state))
3410                 return;
3411
3412         pf->flags |= I40E_FLAG_IN_NETPOLL;
3413         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3414                 for (i = 0; i < vsi->num_q_vectors; i++)
3415                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3416         } else {
3417                 i40e_intr(pf->pdev->irq, netdev);
3418         }
3419         pf->flags &= ~I40E_FLAG_IN_NETPOLL;
3420 }
3421 #endif
3422
3423 /**
3424  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3425  * @pf: the PF being configured
3426  * @pf_q: the PF queue
3427  * @enable: enable or disable state of the queue
3428  *
3429  * This routine will wait for the given Tx queue of the PF to reach the
3430  * enabled or disabled state.
3431  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3432  * multiple retries; else will return 0 in case of success.
3433  **/
3434 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3435 {
3436         int i;
3437         u32 tx_reg;
3438
3439         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3440                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
3441                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3442                         break;
3443
3444                 usleep_range(10, 20);
3445         }
3446         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3447                 return -ETIMEDOUT;
3448
3449         return 0;
3450 }
3451
3452 /**
3453  * i40e_vsi_control_tx - Start or stop a VSI's rings
3454  * @vsi: the VSI being configured
3455  * @enable: start or stop the rings
3456  **/
3457 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3458 {
3459         struct i40e_pf *pf = vsi->back;
3460         struct i40e_hw *hw = &pf->hw;
3461         int i, j, pf_q, ret = 0;
3462         u32 tx_reg;
3463
3464         pf_q = vsi->base_queue;
3465         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3466
3467                 /* warn the TX unit of coming changes */
3468                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
3469                 if (!enable)
3470                         usleep_range(10, 20);
3471
3472                 for (j = 0; j < 50; j++) {
3473                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3474                         if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
3475                             ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
3476                                 break;
3477                         usleep_range(1000, 2000);
3478                 }
3479                 /* Skip if the queue is already in the requested state */
3480                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3481                         continue;
3482
3483                 /* turn on/off the queue */
3484                 if (enable) {
3485                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3486                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
3487                 } else {
3488                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3489                 }
3490
3491                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3492
3493                 /* wait for the change to finish */
3494                 ret = i40e_pf_txq_wait(pf, pf_q, enable);
3495                 if (ret) {
3496                         dev_info(&pf->pdev->dev,
3497                                  "%s: VSI seid %d Tx ring %d %sable timeout\n",
3498                                  __func__, vsi->seid, pf_q,
3499                                  (enable ? "en" : "dis"));
3500                         break;
3501                 }
3502         }
3503
3504         if (hw->revision_id == 0)
3505                 mdelay(50);
3506         return ret;
3507 }
3508
3509 /**
3510  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
3511  * @pf: the PF being configured
3512  * @pf_q: the PF queue
3513  * @enable: enable or disable state of the queue
3514  *
3515  * This routine will wait for the given Rx queue of the PF to reach the
3516  * enabled or disabled state.
3517  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3518  * multiple retries; else will return 0 in case of success.
3519  **/
3520 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3521 {
3522         int i;
3523         u32 rx_reg;
3524
3525         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3526                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
3527                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3528                         break;
3529
3530                 usleep_range(10, 20);
3531         }
3532         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3533                 return -ETIMEDOUT;
3534
3535         return 0;
3536 }
3537
3538 /**
3539  * i40e_vsi_control_rx - Start or stop a VSI's rings
3540  * @vsi: the VSI being configured
3541  * @enable: start or stop the rings
3542  **/
3543 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3544 {
3545         struct i40e_pf *pf = vsi->back;
3546         struct i40e_hw *hw = &pf->hw;
3547         int i, j, pf_q, ret = 0;
3548         u32 rx_reg;
3549
3550         pf_q = vsi->base_queue;
3551         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3552                 for (j = 0; j < 50; j++) {
3553                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3554                         if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
3555                             ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
3556                                 break;
3557                         usleep_range(1000, 2000);
3558                 }
3559
3560                 /* Skip if the queue is already in the requested state */
3561                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3562                         continue;
3563
3564                 /* turn on/off the queue */
3565                 if (enable)
3566                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
3567                 else
3568                         rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3569                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3570
3571                 /* wait for the change to finish */
3572                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
3573                 if (ret) {
3574                         dev_info(&pf->pdev->dev,
3575                                  "%s: VSI seid %d Rx ring %d %sable timeout\n",
3576                                  __func__, vsi->seid, pf_q,
3577                                  (enable ? "en" : "dis"));
3578                         break;
3579                 }
3580         }
3581
3582         return ret;
3583 }
3584
3585 /**
3586  * i40e_vsi_control_rings - Start or stop a VSI's rings
3587  * @vsi: the VSI being configured
3588  * @enable: start or stop the rings
3589  **/
3590 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3591 {
3592         int ret = 0;
3593
3594         /* do rx first for enable and last for disable */
3595         if (request) {
3596                 ret = i40e_vsi_control_rx(vsi, request);
3597                 if (ret)
3598                         return ret;
3599                 ret = i40e_vsi_control_tx(vsi, request);
3600         } else {
3601                 /* Ignore return value, we need to shutdown whatever we can */
3602                 i40e_vsi_control_tx(vsi, request);
3603                 i40e_vsi_control_rx(vsi, request);
3604         }
3605
3606         return ret;
3607 }
3608
3609 /**
3610  * i40e_vsi_free_irq - Free the irq association with the OS
3611  * @vsi: the VSI being configured
3612  **/
3613 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3614 {
3615         struct i40e_pf *pf = vsi->back;
3616         struct i40e_hw *hw = &pf->hw;
3617         int base = vsi->base_vector;
3618         u32 val, qp;
3619         int i;
3620
3621         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3622                 if (!vsi->q_vectors)
3623                         return;
3624
3625                 if (!vsi->irqs_ready)
3626                         return;
3627
3628                 vsi->irqs_ready = false;
3629                 for (i = 0; i < vsi->num_q_vectors; i++) {
3630                         u16 vector = i + base;
3631
3632                         /* free only the irqs that were actually requested */
3633                         if (!vsi->q_vectors[i] ||
3634                             !vsi->q_vectors[i]->num_ringpairs)
3635                                 continue;
3636
3637                         /* clear the affinity_mask in the IRQ descriptor */
3638                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
3639                                               NULL);
3640                         free_irq(pf->msix_entries[vector].vector,
3641                                  vsi->q_vectors[i]);
3642
3643                         /* Tear down the interrupt queue link list
3644                          *
3645                          * We know that they come in pairs and always
3646                          * the Rx first, then the Tx.  To clear the
3647                          * link list, stick the EOL value into the
3648                          * next_q field of the registers.
3649                          */
3650                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
3651                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3652                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3653                         val |= I40E_QUEUE_END_OF_LIST
3654                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3655                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
3656
3657                         while (qp != I40E_QUEUE_END_OF_LIST) {
3658                                 u32 next;
3659
3660                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3661
3662                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3663                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3664                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3665                                          I40E_QINT_RQCTL_INTEVENT_MASK);
3666
3667                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3668                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3669
3670                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3671
3672                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3673
3674                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
3675                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
3676
3677                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3678                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3679                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3680                                          I40E_QINT_TQCTL_INTEVENT_MASK);
3681
3682                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3683                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3684
3685                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3686                                 qp = next;
3687                         }
3688                 }
3689         } else {
3690                 free_irq(pf->pdev->irq, pf);
3691
3692                 val = rd32(hw, I40E_PFINT_LNKLST0);
3693                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3694                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3695                 val |= I40E_QUEUE_END_OF_LIST
3696                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
3697                 wr32(hw, I40E_PFINT_LNKLST0, val);
3698
3699                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3700                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3701                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3702                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3703                          I40E_QINT_RQCTL_INTEVENT_MASK);
3704
3705                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3706                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3707
3708                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3709
3710                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3711
3712                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3713                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3714                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3715                          I40E_QINT_TQCTL_INTEVENT_MASK);
3716
3717                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3718                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3719
3720                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3721         }
3722 }
3723
3724 /**
3725  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
3726  * @vsi: the VSI being configured
3727  * @v_idx: Index of vector to be freed
3728  *
3729  * This function frees the memory allocated to the q_vector.  In addition if
3730  * NAPI is enabled it will delete any references to the NAPI struct prior
3731  * to freeing the q_vector.
3732  **/
3733 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
3734 {
3735         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3736         struct i40e_ring *ring;
3737
3738         if (!q_vector)
3739                 return;
3740
3741         /* disassociate q_vector from rings */
3742         i40e_for_each_ring(ring, q_vector->tx)
3743                 ring->q_vector = NULL;
3744
3745         i40e_for_each_ring(ring, q_vector->rx)
3746                 ring->q_vector = NULL;
3747
3748         /* only VSI w/ an associated netdev is set up w/ NAPI */
3749         if (vsi->netdev)
3750                 netif_napi_del(&q_vector->napi);
3751
3752         vsi->q_vectors[v_idx] = NULL;
3753
3754         kfree_rcu(q_vector, rcu);
3755 }
3756
3757 /**
3758  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
3759  * @vsi: the VSI being un-configured
3760  *
3761  * This frees the memory allocated to the q_vectors and
3762  * deletes references to the NAPI struct.
3763  **/
3764 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
3765 {
3766         int v_idx;
3767
3768         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
3769                 i40e_free_q_vector(vsi, v_idx);
3770 }
3771
3772 /**
3773  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
3774  * @pf: board private structure
3775  **/
3776 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
3777 {
3778         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
3779         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3780                 pci_disable_msix(pf->pdev);
3781                 kfree(pf->msix_entries);
3782                 pf->msix_entries = NULL;
3783         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
3784                 pci_disable_msi(pf->pdev);
3785         }
3786         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
3787 }
3788
3789 /**
3790  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
3791  * @pf: board private structure
3792  *
3793  * We go through and clear interrupt specific resources and reset the structure
3794  * to pre-load conditions
3795  **/
3796 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
3797 {
3798         int i;
3799
3800         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
3801         for (i = 0; i < pf->num_alloc_vsi; i++)
3802                 if (pf->vsi[i])
3803                         i40e_vsi_free_q_vectors(pf->vsi[i]);
3804         i40e_reset_interrupt_capability(pf);
3805 }
3806
3807 /**
3808  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
3809  * @vsi: the VSI being configured
3810  **/
3811 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
3812 {
3813         int q_idx;
3814
3815         if (!vsi->netdev)
3816                 return;
3817
3818         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3819                 napi_enable(&vsi->q_vectors[q_idx]->napi);
3820 }
3821
3822 /**
3823  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
3824  * @vsi: the VSI being configured
3825  **/
3826 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
3827 {
3828         int q_idx;
3829
3830         if (!vsi->netdev)
3831                 return;
3832
3833         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3834                 napi_disable(&vsi->q_vectors[q_idx]->napi);
3835 }
3836
3837 /**
3838  * i40e_vsi_close - Shut down a VSI
3839  * @vsi: the vsi to be quelled
3840  **/
3841 static void i40e_vsi_close(struct i40e_vsi *vsi)
3842 {
3843         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
3844                 i40e_down(vsi);
3845         i40e_vsi_free_irq(vsi);
3846         i40e_vsi_free_tx_resources(vsi);
3847         i40e_vsi_free_rx_resources(vsi);
3848 }
3849
3850 /**
3851  * i40e_quiesce_vsi - Pause a given VSI
3852  * @vsi: the VSI being paused
3853  **/
3854 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
3855 {
3856         if (test_bit(__I40E_DOWN, &vsi->state))
3857                 return;
3858
3859         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
3860         if (vsi->netdev && netif_running(vsi->netdev)) {
3861                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
3862         } else {
3863                 i40e_vsi_close(vsi);
3864         }
3865 }
3866
3867 /**
3868  * i40e_unquiesce_vsi - Resume a given VSI
3869  * @vsi: the VSI being resumed
3870  **/
3871 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
3872 {
3873         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
3874                 return;
3875
3876         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
3877         if (vsi->netdev && netif_running(vsi->netdev))
3878                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
3879         else
3880                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
3881 }
3882
3883 /**
3884  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
3885  * @pf: the PF
3886  **/
3887 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
3888 {
3889         int v;
3890
3891         for (v = 0; v < pf->num_alloc_vsi; v++) {
3892                 if (pf->vsi[v])
3893                         i40e_quiesce_vsi(pf->vsi[v]);
3894         }
3895 }
3896
3897 /**
3898  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
3899  * @pf: the PF
3900  **/
3901 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
3902 {
3903         int v;
3904
3905         for (v = 0; v < pf->num_alloc_vsi; v++) {
3906                 if (pf->vsi[v])
3907                         i40e_unquiesce_vsi(pf->vsi[v]);
3908         }
3909 }
3910
3911 /**
3912  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
3913  * @dcbcfg: the corresponding DCBx configuration structure
3914  *
3915  * Return the number of TCs from given DCBx configuration
3916  **/
3917 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
3918 {
3919         u8 num_tc = 0;
3920         int i;
3921
3922         /* Scan the ETS Config Priority Table to find
3923          * traffic class enabled for a given priority
3924          * and use the traffic class index to get the
3925          * number of traffic classes enabled
3926          */
3927         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3928                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
3929                         num_tc = dcbcfg->etscfg.prioritytable[i];
3930         }
3931
3932         /* Traffic class index starts from zero so
3933          * increment to return the actual count
3934          */
3935         return num_tc + 1;
3936 }
3937
3938 /**
3939  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
3940  * @dcbcfg: the corresponding DCBx configuration structure
3941  *
3942  * Query the current DCB configuration and return the number of
3943  * traffic classes enabled from the given DCBX config
3944  **/
3945 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
3946 {
3947         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
3948         u8 enabled_tc = 1;
3949         u8 i;
3950
3951         for (i = 0; i < num_tc; i++)
3952                 enabled_tc |= 1 << i;
3953
3954         return enabled_tc;
3955 }
3956
3957 /**
3958  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
3959  * @pf: PF being queried
3960  *
3961  * Return number of traffic classes enabled for the given PF
3962  **/
3963 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
3964 {
3965         struct i40e_hw *hw = &pf->hw;
3966         u8 i, enabled_tc;
3967         u8 num_tc = 0;
3968         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3969
3970         /* If DCB is not enabled then always in single TC */
3971         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3972                 return 1;
3973
3974         /* MFP mode return count of enabled TCs for this PF */
3975         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3976                 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3977                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3978                         if (enabled_tc & (1 << i))
3979                                 num_tc++;
3980                 }
3981                 return num_tc;
3982         }
3983
3984         /* SFP mode will be enabled for all TCs on port */
3985         return i40e_dcb_get_num_tc(dcbcfg);
3986 }
3987
3988 /**
3989  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
3990  * @pf: PF being queried
3991  *
3992  * Return a bitmap for first enabled traffic class for this PF.
3993  **/
3994 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
3995 {
3996         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3997         u8 i = 0;
3998
3999         if (!enabled_tc)
4000                 return 0x1; /* TC0 */
4001
4002         /* Find the first enabled TC */
4003         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4004                 if (enabled_tc & (1 << i))
4005                         break;
4006         }
4007
4008         return 1 << i;
4009 }
4010
4011 /**
4012  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4013  * @pf: PF being queried
4014  *
4015  * Return a bitmap for enabled traffic classes for this PF.
4016  **/
4017 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4018 {
4019         /* If DCB is not enabled for this PF then just return default TC */
4020         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4021                 return i40e_pf_get_default_tc(pf);
4022
4023         /* MFP mode will have enabled TCs set by FW */
4024         if (pf->flags & I40E_FLAG_MFP_ENABLED)
4025                 return pf->hw.func_caps.enabled_tcmap;
4026
4027         /* SFP mode we want PF to be enabled for all TCs */
4028         return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4029 }
4030
4031 /**
4032  * i40e_vsi_get_bw_info - Query VSI BW Information
4033  * @vsi: the VSI being queried
4034  *
4035  * Returns 0 on success, negative value on failure
4036  **/
4037 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4038 {
4039         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4040         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4041         struct i40e_pf *pf = vsi->back;
4042         struct i40e_hw *hw = &pf->hw;
4043         i40e_status aq_ret;
4044         u32 tc_bw_max;
4045         int i;
4046
4047         /* Get the VSI level BW configuration */
4048         aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4049         if (aq_ret) {
4050                 dev_info(&pf->pdev->dev,
4051                          "couldn't get pf vsi bw config, err %d, aq_err %d\n",
4052                          aq_ret, pf->hw.aq.asq_last_status);
4053                 return -EINVAL;
4054         }
4055
4056         /* Get the VSI level BW configuration per TC */
4057         aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4058                                                   NULL);
4059         if (aq_ret) {
4060                 dev_info(&pf->pdev->dev,
4061                          "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
4062                          aq_ret, pf->hw.aq.asq_last_status);
4063                 return -EINVAL;
4064         }
4065
4066         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4067                 dev_info(&pf->pdev->dev,
4068                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4069                          bw_config.tc_valid_bits,
4070                          bw_ets_config.tc_valid_bits);
4071                 /* Still continuing */
4072         }
4073
4074         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4075         vsi->bw_max_quanta = bw_config.max_bw;
4076         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4077                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4078         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4079                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4080                 vsi->bw_ets_limit_credits[i] =
4081                                         le16_to_cpu(bw_ets_config.credits[i]);
4082                 /* 3 bits out of 4 for each TC */
4083                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4084         }
4085
4086         return 0;
4087 }
4088
4089 /**
4090  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4091  * @vsi: the VSI being configured
4092  * @enabled_tc: TC bitmap
4093  * @bw_credits: BW shared credits per TC
4094  *
4095  * Returns 0 on success, negative value on failure
4096  **/
4097 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4098                                        u8 *bw_share)
4099 {
4100         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4101         i40e_status aq_ret;
4102         int i;
4103
4104         bw_data.tc_valid_bits = enabled_tc;
4105         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4106                 bw_data.tc_bw_credits[i] = bw_share[i];
4107
4108         aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
4109                                           NULL);
4110         if (aq_ret) {
4111                 dev_info(&vsi->back->pdev->dev,
4112                          "AQ command Config VSI BW allocation per TC failed = %d\n",
4113                          vsi->back->hw.aq.asq_last_status);
4114                 return -EINVAL;
4115         }
4116
4117         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4118                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
4119
4120         return 0;
4121 }
4122
4123 /**
4124  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
4125  * @vsi: the VSI being configured
4126  * @enabled_tc: TC map to be enabled
4127  *
4128  **/
4129 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4130 {
4131         struct net_device *netdev = vsi->netdev;
4132         struct i40e_pf *pf = vsi->back;
4133         struct i40e_hw *hw = &pf->hw;
4134         u8 netdev_tc = 0;
4135         int i;
4136         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4137
4138         if (!netdev)
4139                 return;
4140
4141         if (!enabled_tc) {
4142                 netdev_reset_tc(netdev);
4143                 return;
4144         }
4145
4146         /* Set up actual enabled TCs on the VSI */
4147         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
4148                 return;
4149
4150         /* set per TC queues for the VSI */
4151         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4152                 /* Only set TC queues for enabled tcs
4153                  *
4154                  * e.g. For a VSI that has TC0 and TC3 enabled the
4155                  * enabled_tc bitmap would be 0x00001001; the driver
4156                  * will set the numtc for netdev as 2 that will be
4157                  * referenced by the netdev layer as TC 0 and 1.
4158                  */
4159                 if (vsi->tc_config.enabled_tc & (1 << i))
4160                         netdev_set_tc_queue(netdev,
4161                                         vsi->tc_config.tc_info[i].netdev_tc,
4162                                         vsi->tc_config.tc_info[i].qcount,
4163                                         vsi->tc_config.tc_info[i].qoffset);
4164         }
4165
4166         /* Assign UP2TC map for the VSI */
4167         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
4168                 /* Get the actual TC# for the UP */
4169                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
4170                 /* Get the mapped netdev TC# for the UP */
4171                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
4172                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
4173         }
4174 }
4175
4176 /**
4177  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
4178  * @vsi: the VSI being configured
4179  * @ctxt: the ctxt buffer returned from AQ VSI update param command
4180  **/
4181 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
4182                                       struct i40e_vsi_context *ctxt)
4183 {
4184         /* copy just the sections touched not the entire info
4185          * since not all sections are valid as returned by
4186          * update vsi params
4187          */
4188         vsi->info.mapping_flags = ctxt->info.mapping_flags;
4189         memcpy(&vsi->info.queue_mapping,
4190                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
4191         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
4192                sizeof(vsi->info.tc_mapping));
4193 }
4194
4195 /**
4196  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
4197  * @vsi: VSI to be configured
4198  * @enabled_tc: TC bitmap
4199  *
4200  * This configures a particular VSI for TCs that are mapped to the
4201  * given TC bitmap. It uses default bandwidth share for TCs across
4202  * VSIs to configure TC for a particular VSI.
4203  *
4204  * NOTE:
4205  * It is expected that the VSI queues have been quisced before calling
4206  * this function.
4207  **/
4208 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4209 {
4210         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
4211         struct i40e_vsi_context ctxt;
4212         int ret = 0;
4213         int i;
4214
4215         /* Check if enabled_tc is same as existing or new TCs */
4216         if (vsi->tc_config.enabled_tc == enabled_tc)
4217                 return ret;
4218
4219         /* Enable ETS TCs with equal BW Share for now across all VSIs */
4220         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4221                 if (enabled_tc & (1 << i))
4222                         bw_share[i] = 1;
4223         }
4224
4225         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
4226         if (ret) {
4227                 dev_info(&vsi->back->pdev->dev,
4228                          "Failed configuring TC map %d for VSI %d\n",
4229                          enabled_tc, vsi->seid);
4230                 goto out;
4231         }
4232
4233         /* Update Queue Pairs Mapping for currently enabled UPs */
4234         ctxt.seid = vsi->seid;
4235         ctxt.pf_num = vsi->back->hw.pf_id;
4236         ctxt.vf_num = 0;
4237         ctxt.uplink_seid = vsi->uplink_seid;
4238         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4239         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
4240
4241         /* Update the VSI after updating the VSI queue-mapping information */
4242         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
4243         if (ret) {
4244                 dev_info(&vsi->back->pdev->dev,
4245                          "update vsi failed, aq_err=%d\n",
4246                          vsi->back->hw.aq.asq_last_status);
4247                 goto out;
4248         }
4249         /* update the local VSI info with updated queue map */
4250         i40e_vsi_update_queue_map(vsi, &ctxt);
4251         vsi->info.valid_sections = 0;
4252
4253         /* Update current VSI BW information */
4254         ret = i40e_vsi_get_bw_info(vsi);
4255         if (ret) {
4256                 dev_info(&vsi->back->pdev->dev,
4257                          "Failed updating vsi bw info, aq_err=%d\n",
4258                          vsi->back->hw.aq.asq_last_status);
4259                 goto out;
4260         }
4261
4262         /* Update the netdev TC setup */
4263         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
4264 out:
4265         return ret;
4266 }
4267
4268 /**
4269  * i40e_veb_config_tc - Configure TCs for given VEB
4270  * @veb: given VEB
4271  * @enabled_tc: TC bitmap
4272  *
4273  * Configures given TC bitmap for VEB (switching) element
4274  **/
4275 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
4276 {
4277         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
4278         struct i40e_pf *pf = veb->pf;
4279         int ret = 0;
4280         int i;
4281
4282         /* No TCs or already enabled TCs just return */
4283         if (!enabled_tc || veb->enabled_tc == enabled_tc)
4284                 return ret;
4285
4286         bw_data.tc_valid_bits = enabled_tc;
4287         /* bw_data.absolute_credits is not set (relative) */
4288
4289         /* Enable ETS TCs with equal BW Share for now */
4290         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4291                 if (enabled_tc & (1 << i))
4292                         bw_data.tc_bw_share_credits[i] = 1;
4293         }
4294
4295         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
4296                                                    &bw_data, NULL);
4297         if (ret) {
4298                 dev_info(&pf->pdev->dev,
4299                          "veb bw config failed, aq_err=%d\n",
4300                          pf->hw.aq.asq_last_status);
4301                 goto out;
4302         }
4303
4304         /* Update the BW information */
4305         ret = i40e_veb_get_bw_info(veb);
4306         if (ret) {
4307                 dev_info(&pf->pdev->dev,
4308                          "Failed getting veb bw config, aq_err=%d\n",
4309                          pf->hw.aq.asq_last_status);
4310         }
4311
4312 out:
4313         return ret;
4314 }
4315
4316 #ifdef CONFIG_I40E_DCB
4317 /**
4318  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
4319  * @pf: PF struct
4320  *
4321  * Reconfigure VEB/VSIs on a given PF; it is assumed that
4322  * the caller would've quiesce all the VSIs before calling
4323  * this function
4324  **/
4325 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
4326 {
4327         u8 tc_map = 0;
4328         int ret;
4329         u8 v;
4330
4331         /* Enable the TCs available on PF to all VEBs */
4332         tc_map = i40e_pf_get_tc_map(pf);
4333         for (v = 0; v < I40E_MAX_VEB; v++) {
4334                 if (!pf->veb[v])
4335                         continue;
4336                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
4337                 if (ret) {
4338                         dev_info(&pf->pdev->dev,
4339                                  "Failed configuring TC for VEB seid=%d\n",
4340                                  pf->veb[v]->seid);
4341                         /* Will try to configure as many components */
4342                 }
4343         }
4344
4345         /* Update each VSI */
4346         for (v = 0; v < pf->num_alloc_vsi; v++) {
4347                 if (!pf->vsi[v])
4348                         continue;
4349
4350                 /* - Enable all TCs for the LAN VSI
4351 #ifdef I40E_FCOE
4352                  * - For FCoE VSI only enable the TC configured
4353                  *   as per the APP TLV
4354 #endif
4355                  * - For all others keep them at TC0 for now
4356                  */
4357                 if (v == pf->lan_vsi)
4358                         tc_map = i40e_pf_get_tc_map(pf);
4359                 else
4360                         tc_map = i40e_pf_get_default_tc(pf);
4361 #ifdef I40E_FCOE
4362                 if (pf->vsi[v]->type == I40E_VSI_FCOE)
4363                         tc_map = i40e_get_fcoe_tc_map(pf);
4364 #endif /* #ifdef I40E_FCOE */
4365
4366                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
4367                 if (ret) {
4368                         dev_info(&pf->pdev->dev,
4369                                  "Failed configuring TC for VSI seid=%d\n",
4370                                  pf->vsi[v]->seid);
4371                         /* Will try to configure as many components */
4372                 } else {
4373                         /* Re-configure VSI vectors based on updated TC map */
4374                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
4375                         if (pf->vsi[v]->netdev)
4376                                 i40e_dcbnl_set_all(pf->vsi[v]);
4377                 }
4378         }
4379 }
4380
4381 /**
4382  * i40e_init_pf_dcb - Initialize DCB configuration
4383  * @pf: PF being configured
4384  *
4385  * Query the current DCB configuration and cache it
4386  * in the hardware structure
4387  **/
4388 static int i40e_init_pf_dcb(struct i40e_pf *pf)
4389 {
4390         struct i40e_hw *hw = &pf->hw;
4391         int err = 0;
4392
4393         if (pf->hw.func_caps.npar_enable)
4394                 goto out;
4395
4396         /* Get the initial DCB configuration */
4397         err = i40e_init_dcb(hw);
4398         if (!err) {
4399                 /* Device/Function is not DCBX capable */
4400                 if ((!hw->func_caps.dcb) ||
4401                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
4402                         dev_info(&pf->pdev->dev,
4403                                  "DCBX offload is not supported or is disabled for this PF.\n");
4404
4405                         if (pf->flags & I40E_FLAG_MFP_ENABLED)
4406                                 goto out;
4407
4408                 } else {
4409                         /* When status is not DISABLED then DCBX in FW */
4410                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
4411                                        DCB_CAP_DCBX_VER_IEEE;
4412
4413                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
4414                         /* Enable DCB tagging only when more than one TC */
4415                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
4416                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
4417                 }
4418         } else {
4419                 dev_info(&pf->pdev->dev, "AQ Querying DCB configuration failed: %d\n",
4420                          pf->hw.aq.asq_last_status);
4421         }
4422
4423 out:
4424         return err;
4425 }
4426 #endif /* CONFIG_I40E_DCB */
4427 #define SPEED_SIZE 14
4428 #define FC_SIZE 8
4429 /**
4430  * i40e_print_link_message - print link up or down
4431  * @vsi: the VSI for which link needs a message
4432  */
4433 static void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
4434 {
4435         char speed[SPEED_SIZE] = "Unknown";
4436         char fc[FC_SIZE] = "RX/TX";
4437
4438         if (!isup) {
4439                 netdev_info(vsi->netdev, "NIC Link is Down\n");
4440                 return;
4441         }
4442
4443         switch (vsi->back->hw.phy.link_info.link_speed) {
4444         case I40E_LINK_SPEED_40GB:
4445                 strlcpy(speed, "40 Gbps", SPEED_SIZE);
4446                 break;
4447         case I40E_LINK_SPEED_10GB:
4448                 strlcpy(speed, "10 Gbps", SPEED_SIZE);
4449                 break;
4450         case I40E_LINK_SPEED_1GB:
4451                 strlcpy(speed, "1000 Mbps", SPEED_SIZE);
4452                 break;
4453         case I40E_LINK_SPEED_100MB:
4454                 strncpy(speed, "100 Mbps", SPEED_SIZE);
4455                 break;
4456         default:
4457                 break;
4458         }
4459
4460         switch (vsi->back->hw.fc.current_mode) {
4461         case I40E_FC_FULL:
4462                 strlcpy(fc, "RX/TX", FC_SIZE);
4463                 break;
4464         case I40E_FC_TX_PAUSE:
4465                 strlcpy(fc, "TX", FC_SIZE);
4466                 break;
4467         case I40E_FC_RX_PAUSE:
4468                 strlcpy(fc, "RX", FC_SIZE);
4469                 break;
4470         default:
4471                 strlcpy(fc, "None", FC_SIZE);
4472                 break;
4473         }
4474
4475         netdev_info(vsi->netdev, "NIC Link is Up %s Full Duplex, Flow Control: %s\n",
4476                     speed, fc);
4477 }
4478
4479 /**
4480  * i40e_up_complete - Finish the last steps of bringing up a connection
4481  * @vsi: the VSI being configured
4482  **/
4483 static int i40e_up_complete(struct i40e_vsi *vsi)
4484 {
4485         struct i40e_pf *pf = vsi->back;
4486         int err;
4487
4488         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4489                 i40e_vsi_configure_msix(vsi);
4490         else
4491                 i40e_configure_msi_and_legacy(vsi);
4492
4493         /* start rings */
4494         err = i40e_vsi_control_rings(vsi, true);
4495         if (err)
4496                 return err;
4497
4498         clear_bit(__I40E_DOWN, &vsi->state);
4499         i40e_napi_enable_all(vsi);
4500         i40e_vsi_enable_irq(vsi);
4501
4502         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
4503             (vsi->netdev)) {
4504                 i40e_print_link_message(vsi, true);
4505                 netif_tx_start_all_queues(vsi->netdev);
4506                 netif_carrier_on(vsi->netdev);
4507         } else if (vsi->netdev) {
4508                 i40e_print_link_message(vsi, false);
4509                 /* need to check for qualified module here*/
4510                 if ((pf->hw.phy.link_info.link_info &
4511                         I40E_AQ_MEDIA_AVAILABLE) &&
4512                     (!(pf->hw.phy.link_info.an_info &
4513                         I40E_AQ_QUALIFIED_MODULE)))
4514                         netdev_err(vsi->netdev,
4515                                    "the driver failed to link because an unqualified module was detected.");
4516         }
4517
4518         /* replay FDIR SB filters */
4519         if (vsi->type == I40E_VSI_FDIR) {
4520                 /* reset fd counters */
4521                 pf->fd_add_err = pf->fd_atr_cnt = 0;
4522                 if (pf->fd_tcp_rule > 0) {
4523                         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
4524                         dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n");
4525                         pf->fd_tcp_rule = 0;
4526                 }
4527                 i40e_fdir_filter_restore(vsi);
4528         }
4529         i40e_service_event_schedule(pf);
4530
4531         return 0;
4532 }
4533
4534 /**
4535  * i40e_vsi_reinit_locked - Reset the VSI
4536  * @vsi: the VSI being configured
4537  *
4538  * Rebuild the ring structs after some configuration
4539  * has changed, e.g. MTU size.
4540  **/
4541 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
4542 {
4543         struct i40e_pf *pf = vsi->back;
4544
4545         WARN_ON(in_interrupt());
4546         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
4547                 usleep_range(1000, 2000);
4548         i40e_down(vsi);
4549
4550         /* Give a VF some time to respond to the reset.  The
4551          * two second wait is based upon the watchdog cycle in
4552          * the VF driver.
4553          */
4554         if (vsi->type == I40E_VSI_SRIOV)
4555                 msleep(2000);
4556         i40e_up(vsi);
4557         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
4558 }
4559
4560 /**
4561  * i40e_up - Bring the connection back up after being down
4562  * @vsi: the VSI being configured
4563  **/
4564 int i40e_up(struct i40e_vsi *vsi)
4565 {
4566         int err;
4567
4568         err = i40e_vsi_configure(vsi);
4569         if (!err)
4570                 err = i40e_up_complete(vsi);
4571
4572         return err;
4573 }
4574
4575 /**
4576  * i40e_down - Shutdown the connection processing
4577  * @vsi: the VSI being stopped
4578  **/
4579 void i40e_down(struct i40e_vsi *vsi)
4580 {
4581         int i;
4582
4583         /* It is assumed that the caller of this function
4584          * sets the vsi->state __I40E_DOWN bit.
4585          */
4586         if (vsi->netdev) {
4587                 netif_carrier_off(vsi->netdev);
4588                 netif_tx_disable(vsi->netdev);
4589         }
4590         i40e_vsi_disable_irq(vsi);
4591         i40e_vsi_control_rings(vsi, false);
4592         i40e_napi_disable_all(vsi);
4593
4594         for (i = 0; i < vsi->num_queue_pairs; i++) {
4595                 i40e_clean_tx_ring(vsi->tx_rings[i]);
4596                 i40e_clean_rx_ring(vsi->rx_rings[i]);
4597         }
4598 }
4599
4600 /**
4601  * i40e_setup_tc - configure multiple traffic classes
4602  * @netdev: net device to configure
4603  * @tc: number of traffic classes to enable
4604  **/
4605 #ifdef I40E_FCOE
4606 int i40e_setup_tc(struct net_device *netdev, u8 tc)
4607 #else
4608 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
4609 #endif
4610 {
4611         struct i40e_netdev_priv *np = netdev_priv(netdev);
4612         struct i40e_vsi *vsi = np->vsi;
4613         struct i40e_pf *pf = vsi->back;
4614         u8 enabled_tc = 0;
4615         int ret = -EINVAL;
4616         int i;
4617
4618         /* Check if DCB enabled to continue */
4619         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
4620                 netdev_info(netdev, "DCB is not enabled for adapter\n");
4621                 goto exit;
4622         }
4623
4624         /* Check if MFP enabled */
4625         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
4626                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
4627                 goto exit;
4628         }
4629
4630         /* Check whether tc count is within enabled limit */
4631         if (tc > i40e_pf_get_num_tc(pf)) {
4632                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
4633                 goto exit;
4634         }
4635
4636         /* Generate TC map for number of tc requested */
4637         for (i = 0; i < tc; i++)
4638                 enabled_tc |= (1 << i);
4639
4640         /* Requesting same TC configuration as already enabled */
4641         if (enabled_tc == vsi->tc_config.enabled_tc)
4642                 return 0;
4643
4644         /* Quiesce VSI queues */
4645         i40e_quiesce_vsi(vsi);
4646
4647         /* Configure VSI for enabled TCs */
4648         ret = i40e_vsi_config_tc(vsi, enabled_tc);
4649         if (ret) {
4650                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
4651                             vsi->seid);
4652                 goto exit;
4653         }
4654
4655         /* Unquiesce VSI */
4656         i40e_unquiesce_vsi(vsi);
4657
4658 exit:
4659         return ret;
4660 }
4661
4662 /**
4663  * i40e_open - Called when a network interface is made active
4664  * @netdev: network interface device structure
4665  *
4666  * The open entry point is called when a network interface is made
4667  * active by the system (IFF_UP).  At this point all resources needed
4668  * for transmit and receive operations are allocated, the interrupt
4669  * handler is registered with the OS, the netdev watchdog subtask is
4670  * enabled, and the stack is notified that the interface is ready.
4671  *
4672  * Returns 0 on success, negative value on failure
4673  **/
4674 #ifdef I40E_FCOE
4675 int i40e_open(struct net_device *netdev)
4676 #else
4677 static int i40e_open(struct net_device *netdev)
4678 #endif
4679 {
4680         struct i40e_netdev_priv *np = netdev_priv(netdev);
4681         struct i40e_vsi *vsi = np->vsi;
4682         struct i40e_pf *pf = vsi->back;
4683         int err;
4684
4685         /* disallow open during test or if eeprom is broken */
4686         if (test_bit(__I40E_TESTING, &pf->state) ||
4687             test_bit(__I40E_BAD_EEPROM, &pf->state))
4688                 return -EBUSY;
4689
4690         netif_carrier_off(netdev);
4691
4692         err = i40e_vsi_open(vsi);
4693         if (err)
4694                 return err;
4695
4696         /* configure global TSO hardware offload settings */
4697         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
4698                                                        TCP_FLAG_FIN) >> 16);
4699         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
4700                                                        TCP_FLAG_FIN |
4701                                                        TCP_FLAG_CWR) >> 16);
4702         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
4703
4704 #ifdef CONFIG_I40E_VXLAN
4705         vxlan_get_rx_port(netdev);
4706 #endif
4707
4708         return 0;
4709 }
4710
4711 /**
4712  * i40e_vsi_open -
4713  * @vsi: the VSI to open
4714  *
4715  * Finish initialization of the VSI.
4716  *
4717  * Returns 0 on success, negative value on failure
4718  **/
4719 int i40e_vsi_open(struct i40e_vsi *vsi)
4720 {
4721         struct i40e_pf *pf = vsi->back;
4722         char int_name[IFNAMSIZ];
4723         int err;
4724
4725         /* allocate descriptors */
4726         err = i40e_vsi_setup_tx_resources(vsi);
4727         if (err)
4728                 goto err_setup_tx;
4729         err = i40e_vsi_setup_rx_resources(vsi);
4730         if (err)
4731                 goto err_setup_rx;
4732
4733         err = i40e_vsi_configure(vsi);
4734         if (err)
4735                 goto err_setup_rx;
4736
4737         if (vsi->netdev) {
4738                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4739                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
4740                 err = i40e_vsi_request_irq(vsi, int_name);
4741                 if (err)
4742                         goto err_setup_rx;
4743
4744                 /* Notify the stack of the actual queue counts. */
4745                 err = netif_set_real_num_tx_queues(vsi->netdev,
4746                                                    vsi->num_queue_pairs);
4747                 if (err)
4748                         goto err_set_queues;
4749
4750                 err = netif_set_real_num_rx_queues(vsi->netdev,
4751                                                    vsi->num_queue_pairs);
4752                 if (err)
4753                         goto err_set_queues;
4754
4755         } else if (vsi->type == I40E_VSI_FDIR) {
4756                 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
4757                          dev_driver_string(&pf->pdev->dev));
4758                 err = i40e_vsi_request_irq(vsi, int_name);
4759         } else {
4760                 err = -EINVAL;
4761                 goto err_setup_rx;
4762         }
4763
4764         err = i40e_up_complete(vsi);
4765         if (err)
4766                 goto err_up_complete;
4767
4768         return 0;
4769
4770 err_up_complete:
4771         i40e_down(vsi);
4772 err_set_queues:
4773         i40e_vsi_free_irq(vsi);
4774 err_setup_rx:
4775         i40e_vsi_free_rx_resources(vsi);
4776 err_setup_tx:
4777         i40e_vsi_free_tx_resources(vsi);
4778         if (vsi == pf->vsi[pf->lan_vsi])
4779                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
4780
4781         return err;
4782 }
4783
4784 /**
4785  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
4786  * @pf: Pointer to pf
4787  *
4788  * This function destroys the hlist where all the Flow Director
4789  * filters were saved.
4790  **/
4791 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
4792 {
4793         struct i40e_fdir_filter *filter;
4794         struct hlist_node *node2;
4795
4796         hlist_for_each_entry_safe(filter, node2,
4797                                   &pf->fdir_filter_list, fdir_node) {
4798                 hlist_del(&filter->fdir_node);
4799                 kfree(filter);
4800         }
4801         pf->fdir_pf_active_filters = 0;
4802 }
4803
4804 /**
4805  * i40e_close - Disables a network interface
4806  * @netdev: network interface device structure
4807  *
4808  * The close entry point is called when an interface is de-activated
4809  * by the OS.  The hardware is still under the driver's control, but
4810  * this netdev interface is disabled.
4811  *
4812  * Returns 0, this is not allowed to fail
4813  **/
4814 #ifdef I40E_FCOE
4815 int i40e_close(struct net_device *netdev)
4816 #else
4817 static int i40e_close(struct net_device *netdev)
4818 #endif
4819 {
4820         struct i40e_netdev_priv *np = netdev_priv(netdev);
4821         struct i40e_vsi *vsi = np->vsi;
4822
4823         i40e_vsi_close(vsi);
4824
4825         return 0;
4826 }
4827
4828 /**
4829  * i40e_do_reset - Start a PF or Core Reset sequence
4830  * @pf: board private structure
4831  * @reset_flags: which reset is requested
4832  *
4833  * The essential difference in resets is that the PF Reset
4834  * doesn't clear the packet buffers, doesn't reset the PE
4835  * firmware, and doesn't bother the other PFs on the chip.
4836  **/
4837 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
4838 {
4839         u32 val;
4840
4841         WARN_ON(in_interrupt());
4842
4843         if (i40e_check_asq_alive(&pf->hw))
4844                 i40e_vc_notify_reset(pf);
4845
4846         /* do the biggest reset indicated */
4847         if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
4848
4849                 /* Request a Global Reset
4850                  *
4851                  * This will start the chip's countdown to the actual full
4852                  * chip reset event, and a warning interrupt to be sent
4853                  * to all PFs, including the requestor.  Our handler
4854                  * for the warning interrupt will deal with the shutdown
4855                  * and recovery of the switch setup.
4856                  */
4857                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
4858                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4859                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
4860                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4861
4862         } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
4863
4864                 /* Request a Core Reset
4865                  *
4866                  * Same as Global Reset, except does *not* include the MAC/PHY
4867                  */
4868                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
4869                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4870                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
4871                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4872                 i40e_flush(&pf->hw);
4873
4874         } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
4875
4876                 /* Request a Firmware Reset
4877                  *
4878                  * Same as Global reset, plus restarting the
4879                  * embedded firmware engine.
4880                  */
4881                 /* enable EMP Reset */
4882                 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
4883                 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
4884                 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
4885
4886                 /* force the reset */
4887                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4888                 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
4889                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4890                 i40e_flush(&pf->hw);
4891
4892         } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
4893
4894                 /* Request a PF Reset
4895                  *
4896                  * Resets only the PF-specific registers
4897                  *
4898                  * This goes directly to the tear-down and rebuild of
4899                  * the switch, since we need to do all the recovery as
4900                  * for the Core Reset.
4901                  */
4902                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
4903                 i40e_handle_reset_warning(pf);
4904
4905         } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
4906                 int v;
4907
4908                 /* Find the VSI(s) that requested a re-init */
4909                 dev_info(&pf->pdev->dev,
4910                          "VSI reinit requested\n");
4911                 for (v = 0; v < pf->num_alloc_vsi; v++) {
4912                         struct i40e_vsi *vsi = pf->vsi[v];
4913                         if (vsi != NULL &&
4914                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
4915                                 i40e_vsi_reinit_locked(pf->vsi[v]);
4916                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
4917                         }
4918                 }
4919
4920                 /* no further action needed, so return now */
4921                 return;
4922         } else if (reset_flags & (1 << __I40E_DOWN_REQUESTED)) {
4923                 int v;
4924
4925                 /* Find the VSI(s) that needs to be brought down */
4926                 dev_info(&pf->pdev->dev, "VSI down requested\n");
4927                 for (v = 0; v < pf->num_alloc_vsi; v++) {
4928                         struct i40e_vsi *vsi = pf->vsi[v];
4929                         if (vsi != NULL &&
4930                             test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) {
4931                                 set_bit(__I40E_DOWN, &vsi->state);
4932                                 i40e_down(vsi);
4933                                 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state);
4934                         }
4935                 }
4936
4937                 /* no further action needed, so return now */
4938                 return;
4939         } else {
4940                 dev_info(&pf->pdev->dev,
4941                          "bad reset request 0x%08x\n", reset_flags);
4942                 return;
4943         }
4944 }
4945
4946 #ifdef CONFIG_I40E_DCB
4947 /**
4948  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
4949  * @pf: board private structure
4950  * @old_cfg: current DCB config
4951  * @new_cfg: new DCB config
4952  **/
4953 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
4954                             struct i40e_dcbx_config *old_cfg,
4955                             struct i40e_dcbx_config *new_cfg)
4956 {
4957         bool need_reconfig = false;
4958
4959         /* Check if ETS configuration has changed */
4960         if (memcmp(&new_cfg->etscfg,
4961                    &old_cfg->etscfg,
4962                    sizeof(new_cfg->etscfg))) {
4963                 /* If Priority Table has changed reconfig is needed */
4964                 if (memcmp(&new_cfg->etscfg.prioritytable,
4965                            &old_cfg->etscfg.prioritytable,
4966                            sizeof(new_cfg->etscfg.prioritytable))) {
4967                         need_reconfig = true;
4968                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
4969                 }
4970
4971                 if (memcmp(&new_cfg->etscfg.tcbwtable,
4972                            &old_cfg->etscfg.tcbwtable,
4973                            sizeof(new_cfg->etscfg.tcbwtable)))
4974                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
4975
4976                 if (memcmp(&new_cfg->etscfg.tsatable,
4977                            &old_cfg->etscfg.tsatable,
4978                            sizeof(new_cfg->etscfg.tsatable)))
4979                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
4980         }
4981
4982         /* Check if PFC configuration has changed */
4983         if (memcmp(&new_cfg->pfc,
4984                    &old_cfg->pfc,
4985                    sizeof(new_cfg->pfc))) {
4986                 need_reconfig = true;
4987                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
4988         }
4989
4990         /* Check if APP Table has changed */
4991         if (memcmp(&new_cfg->app,
4992                    &old_cfg->app,
4993                    sizeof(new_cfg->app))) {
4994                 need_reconfig = true;
4995                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
4996         }
4997
4998         return need_reconfig;
4999 }
5000
5001 /**
5002  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5003  * @pf: board private structure
5004  * @e: event info posted on ARQ
5005  **/
5006 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5007                                   struct i40e_arq_event_info *e)
5008 {
5009         struct i40e_aqc_lldp_get_mib *mib =
5010                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5011         struct i40e_hw *hw = &pf->hw;
5012         struct i40e_dcbx_config *dcbx_cfg = &hw->local_dcbx_config;
5013         struct i40e_dcbx_config tmp_dcbx_cfg;
5014         bool need_reconfig = false;
5015         int ret = 0;
5016         u8 type;
5017
5018         /* Not DCB capable or capability disabled */
5019         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
5020                 return ret;
5021
5022         /* Ignore if event is not for Nearest Bridge */
5023         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
5024                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
5025         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
5026                 return ret;
5027
5028         /* Check MIB Type and return if event for Remote MIB update */
5029         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
5030         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
5031                 /* Update the remote cached instance and return */
5032                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
5033                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
5034                                 &hw->remote_dcbx_config);
5035                 goto exit;
5036         }
5037
5038         /* Convert/store the DCBX data from LLDPDU temporarily */
5039         memset(&tmp_dcbx_cfg, 0, sizeof(tmp_dcbx_cfg));
5040         ret = i40e_lldp_to_dcb_config(e->msg_buf, &tmp_dcbx_cfg);
5041         if (ret) {
5042                 /* Error in LLDPDU parsing return */
5043                 dev_info(&pf->pdev->dev, "Failed parsing LLDPDU from event buffer\n");
5044                 goto exit;
5045         }
5046
5047         /* No change detected in DCBX configs */
5048         if (!memcmp(&tmp_dcbx_cfg, dcbx_cfg, sizeof(tmp_dcbx_cfg))) {
5049                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
5050                 goto exit;
5051         }
5052
5053         need_reconfig = i40e_dcb_need_reconfig(pf, dcbx_cfg, &tmp_dcbx_cfg);
5054
5055         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg);
5056
5057         /* Overwrite the new configuration */
5058         *dcbx_cfg = tmp_dcbx_cfg;
5059
5060         if (!need_reconfig)
5061                 goto exit;
5062
5063         /* Enable DCB tagging only when more than one TC */
5064         if (i40e_dcb_get_num_tc(dcbx_cfg) > 1)
5065                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5066         else
5067                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5068
5069         /* Reconfiguration needed quiesce all VSIs */
5070         i40e_pf_quiesce_all_vsi(pf);
5071
5072         /* Changes in configuration update VEB/VSI */
5073         i40e_dcb_reconfigure(pf);
5074
5075         i40e_pf_unquiesce_all_vsi(pf);
5076 exit:
5077         return ret;
5078 }
5079 #endif /* CONFIG_I40E_DCB */
5080
5081 /**
5082  * i40e_do_reset_safe - Protected reset path for userland calls.
5083  * @pf: board private structure
5084  * @reset_flags: which reset is requested
5085  *
5086  **/
5087 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
5088 {
5089         rtnl_lock();
5090         i40e_do_reset(pf, reset_flags);
5091         rtnl_unlock();
5092 }
5093
5094 /**
5095  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
5096  * @pf: board private structure
5097  * @e: event info posted on ARQ
5098  *
5099  * Handler for LAN Queue Overflow Event generated by the firmware for PF
5100  * and VF queues
5101  **/
5102 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
5103                                            struct i40e_arq_event_info *e)
5104 {
5105         struct i40e_aqc_lan_overflow *data =
5106                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
5107         u32 queue = le32_to_cpu(data->prtdcb_rupto);
5108         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
5109         struct i40e_hw *hw = &pf->hw;
5110         struct i40e_vf *vf;
5111         u16 vf_id;
5112
5113         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
5114                 queue, qtx_ctl);
5115
5116         /* Queue belongs to VF, find the VF and issue VF reset */
5117         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
5118             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
5119                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
5120                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
5121                 vf_id -= hw->func_caps.vf_base_id;
5122                 vf = &pf->vf[vf_id];
5123                 i40e_vc_notify_vf_reset(vf);
5124                 /* Allow VF to process pending reset notification */
5125                 msleep(20);
5126                 i40e_reset_vf(vf, false);
5127         }
5128 }
5129
5130 /**
5131  * i40e_service_event_complete - Finish up the service event
5132  * @pf: board private structure
5133  **/
5134 static void i40e_service_event_complete(struct i40e_pf *pf)
5135 {
5136         BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
5137
5138         /* flush memory to make sure state is correct before next watchog */
5139         smp_mb__before_atomic();
5140         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
5141 }
5142
5143 /**
5144  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
5145  * @pf: board private structure
5146  **/
5147 int i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
5148 {
5149         int val, fcnt_prog;
5150
5151         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5152         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
5153         return fcnt_prog;
5154 }
5155
5156 /**
5157  * i40e_get_current_fd_count - Get the count of total FD filters programmed
5158  * @pf: board private structure
5159  **/
5160 int i40e_get_current_fd_count(struct i40e_pf *pf)
5161 {
5162         int val, fcnt_prog;
5163         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5164         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
5165                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
5166                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
5167         return fcnt_prog;
5168 }
5169
5170 /**
5171  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
5172  * @pf: board private structure
5173  **/
5174 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
5175 {
5176         u32 fcnt_prog, fcnt_avail;
5177
5178         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
5179                 return;
5180
5181         /* Check if, FD SB or ATR was auto disabled and if there is enough room
5182          * to re-enable
5183          */
5184         fcnt_prog = i40e_get_cur_guaranteed_fd_count(pf);
5185         fcnt_avail = pf->fdir_pf_filter_count;
5186         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
5187             (pf->fd_add_err == 0) ||
5188             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
5189                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
5190                     (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
5191                         pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
5192                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
5193                 }
5194         }
5195         /* Wait for some more space to be available to turn on ATR */
5196         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
5197                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
5198                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
5199                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5200                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n");
5201                 }
5202         }
5203 }
5204
5205 #define I40E_MIN_FD_FLUSH_INTERVAL 10
5206 /**
5207  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
5208  * @pf: board private structure
5209  **/
5210 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
5211 {
5212         int flush_wait_retry = 50;
5213         int reg;
5214
5215         if (time_after(jiffies, pf->fd_flush_timestamp +
5216                                 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) {
5217                 set_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
5218                 pf->fd_flush_timestamp = jiffies;
5219                 pf->auto_disable_flags |= I40E_FLAG_FD_SB_ENABLED;
5220                 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5221                 /* flush all filters */
5222                 wr32(&pf->hw, I40E_PFQF_CTL_1,
5223                      I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
5224                 i40e_flush(&pf->hw);
5225                 pf->fd_flush_cnt++;
5226                 pf->fd_add_err = 0;
5227                 do {
5228                         /* Check FD flush status every 5-6msec */
5229                         usleep_range(5000, 6000);
5230                         reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
5231                         if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
5232                                 break;
5233                 } while (flush_wait_retry--);
5234                 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
5235                         dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
5236                 } else {
5237                         /* replay sideband filters */
5238                         i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
5239
5240                         pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
5241                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5242                         pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
5243                         clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
5244                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
5245                 }
5246         }
5247 }
5248
5249 /**
5250  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
5251  * @pf: board private structure
5252  **/
5253 int i40e_get_current_atr_cnt(struct i40e_pf *pf)
5254 {
5255         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
5256 }
5257
5258 /* We can see up to 256 filter programming desc in transit if the filters are
5259  * being applied really fast; before we see the first
5260  * filter miss error on Rx queue 0. Accumulating enough error messages before
5261  * reacting will make sure we don't cause flush too often.
5262  */
5263 #define I40E_MAX_FD_PROGRAM_ERROR 256
5264
5265 /**
5266  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
5267  * @pf: board private structure
5268  **/
5269 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
5270 {
5271
5272         /* if interface is down do nothing */
5273         if (test_bit(__I40E_DOWN, &pf->state))
5274                 return;
5275
5276         if ((pf->fd_add_err >= I40E_MAX_FD_PROGRAM_ERROR) &&
5277             (i40e_get_current_atr_cnt(pf) >= pf->fd_atr_cnt) &&
5278             (i40e_get_current_atr_cnt(pf) > pf->fdir_pf_filter_count))
5279                 i40e_fdir_flush_and_replay(pf);
5280
5281         i40e_fdir_check_and_reenable(pf);
5282
5283 }
5284
5285 /**
5286  * i40e_vsi_link_event - notify VSI of a link event
5287  * @vsi: vsi to be notified
5288  * @link_up: link up or down
5289  **/
5290 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
5291 {
5292         if (!vsi || test_bit(__I40E_DOWN, &vsi->state))
5293                 return;
5294
5295         switch (vsi->type) {
5296         case I40E_VSI_MAIN:
5297 #ifdef I40E_FCOE
5298         case I40E_VSI_FCOE:
5299 #endif
5300                 if (!vsi->netdev || !vsi->netdev_registered)
5301                         break;
5302
5303                 if (link_up) {
5304                         netif_carrier_on(vsi->netdev);
5305                         netif_tx_wake_all_queues(vsi->netdev);
5306                 } else {
5307                         netif_carrier_off(vsi->netdev);
5308                         netif_tx_stop_all_queues(vsi->netdev);
5309                 }
5310                 break;
5311
5312         case I40E_VSI_SRIOV:
5313                 break;
5314
5315         case I40E_VSI_VMDQ2:
5316         case I40E_VSI_CTRL:
5317         case I40E_VSI_MIRROR:
5318         default:
5319                 /* there is no notification for other VSIs */
5320                 break;
5321         }
5322 }
5323
5324 /**
5325  * i40e_veb_link_event - notify elements on the veb of a link event
5326  * @veb: veb to be notified
5327  * @link_up: link up or down
5328  **/
5329 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
5330 {
5331         struct i40e_pf *pf;
5332         int i;
5333
5334         if (!veb || !veb->pf)
5335                 return;
5336         pf = veb->pf;
5337
5338         /* depth first... */
5339         for (i = 0; i < I40E_MAX_VEB; i++)
5340                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
5341                         i40e_veb_link_event(pf->veb[i], link_up);
5342
5343         /* ... now the local VSIs */
5344         for (i = 0; i < pf->num_alloc_vsi; i++)
5345                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
5346                         i40e_vsi_link_event(pf->vsi[i], link_up);
5347 }
5348
5349 /**
5350  * i40e_link_event - Update netif_carrier status
5351  * @pf: board private structure
5352  **/
5353 static void i40e_link_event(struct i40e_pf *pf)
5354 {
5355         bool new_link, old_link;
5356
5357         /* set this to force the get_link_status call to refresh state */
5358         pf->hw.phy.get_link_info = true;
5359
5360         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
5361         new_link = i40e_get_link_status(&pf->hw);
5362
5363         if (new_link == old_link &&
5364             new_link == netif_carrier_ok(pf->vsi[pf->lan_vsi]->netdev))
5365                 return;
5366         if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
5367                 i40e_print_link_message(pf->vsi[pf->lan_vsi], new_link);
5368
5369         /* Notify the base of the switch tree connected to
5370          * the link.  Floating VEBs are not notified.
5371          */
5372         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
5373                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
5374         else
5375                 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
5376
5377         if (pf->vf)
5378                 i40e_vc_notify_link_state(pf);
5379
5380         if (pf->flags & I40E_FLAG_PTP)
5381                 i40e_ptp_set_increment(pf);
5382 }
5383
5384 /**
5385  * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
5386  * @pf: board private structure
5387  *
5388  * Set the per-queue flags to request a check for stuck queues in the irq
5389  * clean functions, then force interrupts to be sure the irq clean is called.
5390  **/
5391 static void i40e_check_hang_subtask(struct i40e_pf *pf)
5392 {
5393         int i, v;
5394
5395         /* If we're down or resetting, just bail */
5396         if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
5397                 return;
5398
5399         /* for each VSI/netdev
5400          *     for each Tx queue
5401          *         set the check flag
5402          *     for each q_vector
5403          *         force an interrupt
5404          */
5405         for (v = 0; v < pf->num_alloc_vsi; v++) {
5406                 struct i40e_vsi *vsi = pf->vsi[v];
5407                 int armed = 0;
5408
5409                 if (!pf->vsi[v] ||
5410                     test_bit(__I40E_DOWN, &vsi->state) ||
5411                     (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
5412                         continue;
5413
5414                 for (i = 0; i < vsi->num_queue_pairs; i++) {
5415                         set_check_for_tx_hang(vsi->tx_rings[i]);
5416                         if (test_bit(__I40E_HANG_CHECK_ARMED,
5417                                      &vsi->tx_rings[i]->state))
5418                                 armed++;
5419                 }
5420
5421                 if (armed) {
5422                         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
5423                                 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
5424                                      (I40E_PFINT_DYN_CTL0_INTENA_MASK |
5425                                       I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
5426                         } else {
5427                                 u16 vec = vsi->base_vector - 1;
5428                                 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
5429                                            I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
5430                                 for (i = 0; i < vsi->num_q_vectors; i++, vec++)
5431                                         wr32(&vsi->back->hw,
5432                                              I40E_PFINT_DYN_CTLN(vec), val);
5433                         }
5434                         i40e_flush(&vsi->back->hw);
5435                 }
5436         }
5437 }
5438
5439 /**
5440  * i40e_watchdog_subtask - Check and bring link up
5441  * @pf: board private structure
5442  **/
5443 static void i40e_watchdog_subtask(struct i40e_pf *pf)
5444 {
5445         int i;
5446
5447         /* if interface is down do nothing */
5448         if (test_bit(__I40E_DOWN, &pf->state) ||
5449             test_bit(__I40E_CONFIG_BUSY, &pf->state))
5450                 return;
5451
5452         /* Update the stats for active netdevs so the network stack
5453          * can look at updated numbers whenever it cares to
5454          */
5455         for (i = 0; i < pf->num_alloc_vsi; i++)
5456                 if (pf->vsi[i] && pf->vsi[i]->netdev)
5457                         i40e_update_stats(pf->vsi[i]);
5458
5459         /* Update the stats for the active switching components */
5460         for (i = 0; i < I40E_MAX_VEB; i++)
5461                 if (pf->veb[i])
5462                         i40e_update_veb_stats(pf->veb[i]);
5463
5464         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
5465 }
5466
5467 /**
5468  * i40e_reset_subtask - Set up for resetting the device and driver
5469  * @pf: board private structure
5470  **/
5471 static void i40e_reset_subtask(struct i40e_pf *pf)
5472 {
5473         u32 reset_flags = 0;
5474
5475         rtnl_lock();
5476         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
5477                 reset_flags |= (1 << __I40E_REINIT_REQUESTED);
5478                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
5479         }
5480         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
5481                 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
5482                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5483         }
5484         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
5485                 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
5486                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
5487         }
5488         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
5489                 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
5490                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
5491         }
5492         if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) {
5493                 reset_flags |= (1 << __I40E_DOWN_REQUESTED);
5494                 clear_bit(__I40E_DOWN_REQUESTED, &pf->state);
5495         }
5496
5497         /* If there's a recovery already waiting, it takes
5498          * precedence before starting a new reset sequence.
5499          */
5500         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
5501                 i40e_handle_reset_warning(pf);
5502                 goto unlock;
5503         }
5504
5505         /* If we're already down or resetting, just bail */
5506         if (reset_flags &&
5507             !test_bit(__I40E_DOWN, &pf->state) &&
5508             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
5509                 i40e_do_reset(pf, reset_flags);
5510
5511 unlock:
5512         rtnl_unlock();
5513 }
5514
5515 /**
5516  * i40e_handle_link_event - Handle link event
5517  * @pf: board private structure
5518  * @e: event info posted on ARQ
5519  **/
5520 static void i40e_handle_link_event(struct i40e_pf *pf,
5521                                    struct i40e_arq_event_info *e)
5522 {
5523         struct i40e_hw *hw = &pf->hw;
5524         struct i40e_aqc_get_link_status *status =
5525                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
5526         struct i40e_link_status *hw_link_info = &hw->phy.link_info;
5527
5528         /* save off old link status information */
5529         memcpy(&pf->hw.phy.link_info_old, hw_link_info,
5530                sizeof(pf->hw.phy.link_info_old));
5531
5532         /* Do a new status request to re-enable LSE reporting
5533          * and load new status information into the hw struct
5534          * This completely ignores any state information
5535          * in the ARQ event info, instead choosing to always
5536          * issue the AQ update link status command.
5537          */
5538         i40e_link_event(pf);
5539
5540         /* check for unqualified module, if link is down */
5541         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
5542             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
5543             (!(status->link_info & I40E_AQ_LINK_UP)))
5544                 dev_err(&pf->pdev->dev,
5545                         "The driver failed to link because an unqualified module was detected.\n");
5546 }
5547
5548 /**
5549  * i40e_clean_adminq_subtask - Clean the AdminQ rings
5550  * @pf: board private structure
5551  **/
5552 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
5553 {
5554         struct i40e_arq_event_info event;
5555         struct i40e_hw *hw = &pf->hw;
5556         u16 pending, i = 0;
5557         i40e_status ret;
5558         u16 opcode;
5559         u32 oldval;
5560         u32 val;
5561
5562         /* Do not run clean AQ when PF reset fails */
5563         if (test_bit(__I40E_RESET_FAILED, &pf->state))
5564                 return;
5565
5566         /* check for error indications */
5567         val = rd32(&pf->hw, pf->hw.aq.arq.len);
5568         oldval = val;
5569         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
5570                 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
5571                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
5572         }
5573         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
5574                 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
5575                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
5576         }
5577         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
5578                 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
5579                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
5580         }
5581         if (oldval != val)
5582                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
5583
5584         val = rd32(&pf->hw, pf->hw.aq.asq.len);
5585         oldval = val;
5586         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
5587                 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
5588                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
5589         }
5590         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
5591                 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
5592                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
5593         }
5594         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
5595                 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
5596                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
5597         }
5598         if (oldval != val)
5599                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
5600
5601         event.msg_size = I40E_MAX_AQ_BUF_SIZE;
5602         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
5603         if (!event.msg_buf)
5604                 return;
5605
5606         do {
5607                 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
5608                 ret = i40e_clean_arq_element(hw, &event, &pending);
5609                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
5610                         break;
5611                 else if (ret) {
5612                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
5613                         break;
5614                 }
5615
5616                 opcode = le16_to_cpu(event.desc.opcode);
5617                 switch (opcode) {
5618
5619                 case i40e_aqc_opc_get_link_status:
5620                         i40e_handle_link_event(pf, &event);
5621                         break;
5622                 case i40e_aqc_opc_send_msg_to_pf:
5623                         ret = i40e_vc_process_vf_msg(pf,
5624                                         le16_to_cpu(event.desc.retval),
5625                                         le32_to_cpu(event.desc.cookie_high),
5626                                         le32_to_cpu(event.desc.cookie_low),
5627                                         event.msg_buf,
5628                                         event.msg_size);
5629                         break;
5630                 case i40e_aqc_opc_lldp_update_mib:
5631                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
5632 #ifdef CONFIG_I40E_DCB
5633                         rtnl_lock();
5634                         ret = i40e_handle_lldp_event(pf, &event);
5635                         rtnl_unlock();
5636 #endif /* CONFIG_I40E_DCB */
5637                         break;
5638                 case i40e_aqc_opc_event_lan_overflow:
5639                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
5640                         i40e_handle_lan_overflow_event(pf, &event);
5641                         break;
5642                 case i40e_aqc_opc_send_msg_to_peer:
5643                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
5644                         break;
5645                 default:
5646                         dev_info(&pf->pdev->dev,
5647                                  "ARQ Error: Unknown event 0x%04x received\n",
5648                                  opcode);
5649                         break;
5650                 }
5651         } while (pending && (i++ < pf->adminq_work_limit));
5652
5653         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
5654         /* re-enable Admin queue interrupt cause */
5655         val = rd32(hw, I40E_PFINT_ICR0_ENA);
5656         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
5657         wr32(hw, I40E_PFINT_ICR0_ENA, val);
5658         i40e_flush(hw);
5659
5660         kfree(event.msg_buf);
5661 }
5662
5663 /**
5664  * i40e_verify_eeprom - make sure eeprom is good to use
5665  * @pf: board private structure
5666  **/
5667 static void i40e_verify_eeprom(struct i40e_pf *pf)
5668 {
5669         int err;
5670
5671         err = i40e_diag_eeprom_test(&pf->hw);
5672         if (err) {
5673                 /* retry in case of garbage read */
5674                 err = i40e_diag_eeprom_test(&pf->hw);
5675                 if (err) {
5676                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
5677                                  err);
5678                         set_bit(__I40E_BAD_EEPROM, &pf->state);
5679                 }
5680         }
5681
5682         if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
5683                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
5684                 clear_bit(__I40E_BAD_EEPROM, &pf->state);
5685         }
5686 }
5687
5688 /**
5689  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
5690  * @veb: pointer to the VEB instance
5691  *
5692  * This is a recursive function that first builds the attached VSIs then
5693  * recurses in to build the next layer of VEB.  We track the connections
5694  * through our own index numbers because the seid's from the HW could
5695  * change across the reset.
5696  **/
5697 static int i40e_reconstitute_veb(struct i40e_veb *veb)
5698 {
5699         struct i40e_vsi *ctl_vsi = NULL;
5700         struct i40e_pf *pf = veb->pf;
5701         int v, veb_idx;
5702         int ret;
5703
5704         /* build VSI that owns this VEB, temporarily attached to base VEB */
5705         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
5706                 if (pf->vsi[v] &&
5707                     pf->vsi[v]->veb_idx == veb->idx &&
5708                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
5709                         ctl_vsi = pf->vsi[v];
5710                         break;
5711                 }
5712         }
5713         if (!ctl_vsi) {
5714                 dev_info(&pf->pdev->dev,
5715                          "missing owner VSI for veb_idx %d\n", veb->idx);
5716                 ret = -ENOENT;
5717                 goto end_reconstitute;
5718         }
5719         if (ctl_vsi != pf->vsi[pf->lan_vsi])
5720                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
5721         ret = i40e_add_vsi(ctl_vsi);
5722         if (ret) {
5723                 dev_info(&pf->pdev->dev,
5724                          "rebuild of owner VSI failed: %d\n", ret);
5725                 goto end_reconstitute;
5726         }
5727         i40e_vsi_reset_stats(ctl_vsi);
5728
5729         /* create the VEB in the switch and move the VSI onto the VEB */
5730         ret = i40e_add_veb(veb, ctl_vsi);
5731         if (ret)
5732                 goto end_reconstitute;
5733
5734         /* create the remaining VSIs attached to this VEB */
5735         for (v = 0; v < pf->num_alloc_vsi; v++) {
5736                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
5737                         continue;
5738
5739                 if (pf->vsi[v]->veb_idx == veb->idx) {
5740                         struct i40e_vsi *vsi = pf->vsi[v];
5741                         vsi->uplink_seid = veb->seid;
5742                         ret = i40e_add_vsi(vsi);
5743                         if (ret) {
5744                                 dev_info(&pf->pdev->dev,
5745                                          "rebuild of vsi_idx %d failed: %d\n",
5746                                          v, ret);
5747                                 goto end_reconstitute;
5748                         }
5749                         i40e_vsi_reset_stats(vsi);
5750                 }
5751         }
5752
5753         /* create any VEBs attached to this VEB - RECURSION */
5754         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
5755                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
5756                         pf->veb[veb_idx]->uplink_seid = veb->seid;
5757                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
5758                         if (ret)
5759                                 break;
5760                 }
5761         }
5762
5763 end_reconstitute:
5764         return ret;
5765 }
5766
5767 /**
5768  * i40e_get_capabilities - get info about the HW
5769  * @pf: the PF struct
5770  **/
5771 static int i40e_get_capabilities(struct i40e_pf *pf)
5772 {
5773         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
5774         u16 data_size;
5775         int buf_len;
5776         int err;
5777
5778         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
5779         do {
5780                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
5781                 if (!cap_buf)
5782                         return -ENOMEM;
5783
5784                 /* this loads the data into the hw struct for us */
5785                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
5786                                             &data_size,
5787                                             i40e_aqc_opc_list_func_capabilities,
5788                                             NULL);
5789                 /* data loaded, buffer no longer needed */
5790                 kfree(cap_buf);
5791
5792                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
5793                         /* retry with a larger buffer */
5794                         buf_len = data_size;
5795                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
5796                         dev_info(&pf->pdev->dev,
5797                                  "capability discovery failed: aq=%d\n",
5798                                  pf->hw.aq.asq_last_status);
5799                         return -ENODEV;
5800                 }
5801         } while (err);
5802
5803         if (((pf->hw.aq.fw_maj_ver == 2) && (pf->hw.aq.fw_min_ver < 22)) ||
5804             (pf->hw.aq.fw_maj_ver < 2)) {
5805                 pf->hw.func_caps.num_msix_vectors++;
5806                 pf->hw.func_caps.num_msix_vectors_vf++;
5807         }
5808
5809         if (pf->hw.debug_mask & I40E_DEBUG_USER)
5810                 dev_info(&pf->pdev->dev,
5811                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
5812                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
5813                          pf->hw.func_caps.num_msix_vectors,
5814                          pf->hw.func_caps.num_msix_vectors_vf,
5815                          pf->hw.func_caps.fd_filters_guaranteed,
5816                          pf->hw.func_caps.fd_filters_best_effort,
5817                          pf->hw.func_caps.num_tx_qp,
5818                          pf->hw.func_caps.num_vsis);
5819
5820 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
5821                        + pf->hw.func_caps.num_vfs)
5822         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
5823                 dev_info(&pf->pdev->dev,
5824                          "got num_vsis %d, setting num_vsis to %d\n",
5825                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
5826                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
5827         }
5828
5829         return 0;
5830 }
5831
5832 static int i40e_vsi_clear(struct i40e_vsi *vsi);
5833
5834 /**
5835  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
5836  * @pf: board private structure
5837  **/
5838 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
5839 {
5840         struct i40e_vsi *vsi;
5841         int i;
5842
5843         /* quick workaround for an NVM issue that leaves a critical register
5844          * uninitialized
5845          */
5846         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
5847                 static const u32 hkey[] = {
5848                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
5849                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
5850                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
5851                         0x95b3a76d};
5852
5853                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
5854                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
5855         }
5856
5857         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
5858                 return;
5859
5860         /* find existing VSI and see if it needs configuring */
5861         vsi = NULL;
5862         for (i = 0; i < pf->num_alloc_vsi; i++) {
5863                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
5864                         vsi = pf->vsi[i];
5865                         break;
5866                 }
5867         }
5868
5869         /* create a new VSI if none exists */
5870         if (!vsi) {
5871                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
5872                                      pf->vsi[pf->lan_vsi]->seid, 0);
5873                 if (!vsi) {
5874                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
5875                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
5876                         return;
5877                 }
5878         }
5879
5880         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
5881 }
5882
5883 /**
5884  * i40e_fdir_teardown - release the Flow Director resources
5885  * @pf: board private structure
5886  **/
5887 static void i40e_fdir_teardown(struct i40e_pf *pf)
5888 {
5889         int i;
5890
5891         i40e_fdir_filter_exit(pf);
5892         for (i = 0; i < pf->num_alloc_vsi; i++) {
5893                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
5894                         i40e_vsi_release(pf->vsi[i]);
5895                         break;
5896                 }
5897         }
5898 }
5899
5900 /**
5901  * i40e_prep_for_reset - prep for the core to reset
5902  * @pf: board private structure
5903  *
5904  * Close up the VFs and other things in prep for pf Reset.
5905   **/
5906 static void i40e_prep_for_reset(struct i40e_pf *pf)
5907 {
5908         struct i40e_hw *hw = &pf->hw;
5909         i40e_status ret = 0;
5910         u32 v;
5911
5912         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
5913         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
5914                 return;
5915
5916         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
5917
5918         /* quiesce the VSIs and their queues that are not already DOWN */
5919         i40e_pf_quiesce_all_vsi(pf);
5920
5921         for (v = 0; v < pf->num_alloc_vsi; v++) {
5922                 if (pf->vsi[v])
5923                         pf->vsi[v]->seid = 0;
5924         }
5925
5926         i40e_shutdown_adminq(&pf->hw);
5927
5928         /* call shutdown HMC */
5929         if (hw->hmc.hmc_obj) {
5930                 ret = i40e_shutdown_lan_hmc(hw);
5931                 if (ret)
5932                         dev_warn(&pf->pdev->dev,
5933                                  "shutdown_lan_hmc failed: %d\n", ret);
5934         }
5935 }
5936
5937 /**
5938  * i40e_send_version - update firmware with driver version
5939  * @pf: PF struct
5940  */
5941 static void i40e_send_version(struct i40e_pf *pf)
5942 {
5943         struct i40e_driver_version dv;
5944
5945         dv.major_version = DRV_VERSION_MAJOR;
5946         dv.minor_version = DRV_VERSION_MINOR;
5947         dv.build_version = DRV_VERSION_BUILD;
5948         dv.subbuild_version = 0;
5949         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
5950         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
5951 }
5952
5953 /**
5954  * i40e_reset_and_rebuild - reset and rebuild using a saved config
5955  * @pf: board private structure
5956  * @reinit: if the Main VSI needs to re-initialized.
5957  **/
5958 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
5959 {
5960         struct i40e_hw *hw = &pf->hw;
5961         u8 set_fc_aq_fail = 0;
5962         i40e_status ret;
5963         u32 v;
5964
5965         /* Now we wait for GRST to settle out.
5966          * We don't have to delete the VEBs or VSIs from the hw switch
5967          * because the reset will make them disappear.
5968          */
5969         ret = i40e_pf_reset(hw);
5970         if (ret) {
5971                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
5972                 set_bit(__I40E_RESET_FAILED, &pf->state);
5973                 goto clear_recovery;
5974         }
5975         pf->pfr_count++;
5976
5977         if (test_bit(__I40E_DOWN, &pf->state))
5978                 goto clear_recovery;
5979         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
5980
5981         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
5982         ret = i40e_init_adminq(&pf->hw);
5983         if (ret) {
5984                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
5985                 goto clear_recovery;
5986         }
5987
5988         /* re-verify the eeprom if we just had an EMP reset */
5989         if (test_bit(__I40E_EMP_RESET_REQUESTED, &pf->state)) {
5990                 clear_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
5991                 i40e_verify_eeprom(pf);
5992         }
5993
5994         i40e_clear_pxe_mode(hw);
5995         ret = i40e_get_capabilities(pf);
5996         if (ret) {
5997                 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
5998                          ret);
5999                 goto end_core_reset;
6000         }
6001
6002         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
6003                                 hw->func_caps.num_rx_qp,
6004                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
6005         if (ret) {
6006                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
6007                 goto end_core_reset;
6008         }
6009         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
6010         if (ret) {
6011                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
6012                 goto end_core_reset;
6013         }
6014
6015 #ifdef CONFIG_I40E_DCB
6016         ret = i40e_init_pf_dcb(pf);
6017         if (ret) {
6018                 dev_info(&pf->pdev->dev, "init_pf_dcb failed: %d\n", ret);
6019                 goto end_core_reset;
6020         }
6021 #endif /* CONFIG_I40E_DCB */
6022 #ifdef I40E_FCOE
6023         ret = i40e_init_pf_fcoe(pf);
6024         if (ret)
6025                 dev_info(&pf->pdev->dev, "init_pf_fcoe failed: %d\n", ret);
6026
6027 #endif
6028         /* do basic switch setup */
6029         ret = i40e_setup_pf_switch(pf, reinit);
6030         if (ret)
6031                 goto end_core_reset;
6032
6033         /* driver is only interested in link up/down and module qualification
6034          * reports from firmware
6035          */
6036         ret = i40e_aq_set_phy_int_mask(&pf->hw,
6037                                        I40E_AQ_EVENT_LINK_UPDOWN |
6038                                        I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
6039         if (ret)
6040                 dev_info(&pf->pdev->dev, "set phy mask fail, aq_err %d\n", ret);
6041
6042         /* make sure our flow control settings are restored */
6043         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
6044         if (ret)
6045                 dev_info(&pf->pdev->dev, "set fc fail, aq_err %d\n", ret);
6046
6047         /* Rebuild the VSIs and VEBs that existed before reset.
6048          * They are still in our local switch element arrays, so only
6049          * need to rebuild the switch model in the HW.
6050          *
6051          * If there were VEBs but the reconstitution failed, we'll try
6052          * try to recover minimal use by getting the basic PF VSI working.
6053          */
6054         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
6055                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
6056                 /* find the one VEB connected to the MAC, and find orphans */
6057                 for (v = 0; v < I40E_MAX_VEB; v++) {
6058                         if (!pf->veb[v])
6059                                 continue;
6060
6061                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
6062                             pf->veb[v]->uplink_seid == 0) {
6063                                 ret = i40e_reconstitute_veb(pf->veb[v]);
6064
6065                                 if (!ret)
6066                                         continue;
6067
6068                                 /* If Main VEB failed, we're in deep doodoo,
6069                                  * so give up rebuilding the switch and set up
6070                                  * for minimal rebuild of PF VSI.
6071                                  * If orphan failed, we'll report the error
6072                                  * but try to keep going.
6073                                  */
6074                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
6075                                         dev_info(&pf->pdev->dev,
6076                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
6077                                                  ret);
6078                                         pf->vsi[pf->lan_vsi]->uplink_seid
6079                                                                 = pf->mac_seid;
6080                                         break;
6081                                 } else if (pf->veb[v]->uplink_seid == 0) {
6082                                         dev_info(&pf->pdev->dev,
6083                                                  "rebuild of orphan VEB failed: %d\n",
6084                                                  ret);
6085                                 }
6086                         }
6087                 }
6088         }
6089
6090         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
6091                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
6092                 /* no VEB, so rebuild only the Main VSI */
6093                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
6094                 if (ret) {
6095                         dev_info(&pf->pdev->dev,
6096                                  "rebuild of Main VSI failed: %d\n", ret);
6097                         goto end_core_reset;
6098                 }
6099         }
6100
6101         msleep(75);
6102         ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
6103         if (ret) {
6104                 dev_info(&pf->pdev->dev, "link restart failed, aq_err=%d\n",
6105                          pf->hw.aq.asq_last_status);
6106         }
6107
6108         /* reinit the misc interrupt */
6109         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6110                 ret = i40e_setup_misc_vector(pf);
6111
6112         /* restart the VSIs that were rebuilt and running before the reset */
6113         i40e_pf_unquiesce_all_vsi(pf);
6114
6115         if (pf->num_alloc_vfs) {
6116                 for (v = 0; v < pf->num_alloc_vfs; v++)
6117                         i40e_reset_vf(&pf->vf[v], true);
6118         }
6119
6120         /* tell the firmware that we're starting */
6121         i40e_send_version(pf);
6122
6123 end_core_reset:
6124         clear_bit(__I40E_RESET_FAILED, &pf->state);
6125 clear_recovery:
6126         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
6127 }
6128
6129 /**
6130  * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
6131  * @pf: board private structure
6132  *
6133  * Close up the VFs and other things in prep for a Core Reset,
6134  * then get ready to rebuild the world.
6135  **/
6136 static void i40e_handle_reset_warning(struct i40e_pf *pf)
6137 {
6138         i40e_prep_for_reset(pf);
6139         i40e_reset_and_rebuild(pf, false);
6140 }
6141
6142 /**
6143  * i40e_handle_mdd_event
6144  * @pf: pointer to the pf structure
6145  *
6146  * Called from the MDD irq handler to identify possibly malicious vfs
6147  **/
6148 static void i40e_handle_mdd_event(struct i40e_pf *pf)
6149 {
6150         struct i40e_hw *hw = &pf->hw;
6151         bool mdd_detected = false;
6152         bool pf_mdd_detected = false;
6153         struct i40e_vf *vf;
6154         u32 reg;
6155         int i;
6156
6157         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
6158                 return;
6159
6160         /* find what triggered the MDD event */
6161         reg = rd32(hw, I40E_GL_MDET_TX);
6162         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
6163                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
6164                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
6165                 u8 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
6166                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
6167                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT) >>
6168                                 I40E_GL_MDET_TX_EVENT_SHIFT;
6169                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
6170                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
6171                 if (netif_msg_tx_err(pf))
6172                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d pf number 0x%02x vf number 0x%02x\n",
6173                                  event, queue, pf_num, vf_num);
6174                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
6175                 mdd_detected = true;
6176         }
6177         reg = rd32(hw, I40E_GL_MDET_RX);
6178         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
6179                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
6180                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
6181                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT) >>
6182                                 I40E_GL_MDET_RX_EVENT_SHIFT;
6183                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
6184                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
6185                 if (netif_msg_rx_err(pf))
6186                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
6187                                  event, queue, func);
6188                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
6189                 mdd_detected = true;
6190         }
6191
6192         if (mdd_detected) {
6193                 reg = rd32(hw, I40E_PF_MDET_TX);
6194                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
6195                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
6196                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
6197                         pf_mdd_detected = true;
6198                 }
6199                 reg = rd32(hw, I40E_PF_MDET_RX);
6200                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
6201                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
6202                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
6203                         pf_mdd_detected = true;
6204                 }
6205                 /* Queue belongs to the PF, initiate a reset */
6206                 if (pf_mdd_detected) {
6207                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
6208                         i40e_service_event_schedule(pf);
6209                 }
6210         }
6211
6212         /* see if one of the VFs needs its hand slapped */
6213         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
6214                 vf = &(pf->vf[i]);
6215                 reg = rd32(hw, I40E_VP_MDET_TX(i));
6216                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
6217                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
6218                         vf->num_mdd_events++;
6219                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
6220                                  i);
6221                 }
6222
6223                 reg = rd32(hw, I40E_VP_MDET_RX(i));
6224                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
6225                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
6226                         vf->num_mdd_events++;
6227                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
6228                                  i);
6229                 }
6230
6231                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
6232                         dev_info(&pf->pdev->dev,
6233                                  "Too many MDD events on VF %d, disabled\n", i);
6234                         dev_info(&pf->pdev->dev,
6235                                  "Use PF Control I/F to re-enable the VF\n");
6236                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
6237                 }
6238         }
6239
6240         /* re-enable mdd interrupt cause */
6241         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
6242         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
6243         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
6244         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
6245         i40e_flush(hw);
6246 }
6247
6248 #ifdef CONFIG_I40E_VXLAN
6249 /**
6250  * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
6251  * @pf: board private structure
6252  **/
6253 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
6254 {
6255         struct i40e_hw *hw = &pf->hw;
6256         i40e_status ret;
6257         u8 filter_index;
6258         __be16 port;
6259         int i;
6260
6261         if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
6262                 return;
6263
6264         pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
6265
6266         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6267                 if (pf->pending_vxlan_bitmap & (1 << i)) {
6268                         pf->pending_vxlan_bitmap &= ~(1 << i);
6269                         port = pf->vxlan_ports[i];
6270                         ret = port ?
6271                               i40e_aq_add_udp_tunnel(hw, ntohs(port),
6272                                                      I40E_AQC_TUNNEL_TYPE_VXLAN,
6273                                                      &filter_index, NULL)
6274                               : i40e_aq_del_udp_tunnel(hw, i, NULL);
6275
6276                         if (ret) {
6277                                 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
6278                                          port ? "adding" : "deleting",
6279                                          ntohs(port), port ? i : i);
6280
6281                                 pf->vxlan_ports[i] = 0;
6282                         } else {
6283                                 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
6284                                          port ? "Added" : "Deleted",
6285                                          ntohs(port), port ? i : filter_index);
6286                         }
6287                 }
6288         }
6289 }
6290
6291 #endif
6292 /**
6293  * i40e_service_task - Run the driver's async subtasks
6294  * @work: pointer to work_struct containing our data
6295  **/
6296 static void i40e_service_task(struct work_struct *work)
6297 {
6298         struct i40e_pf *pf = container_of(work,
6299                                           struct i40e_pf,
6300                                           service_task);
6301         unsigned long start_time = jiffies;
6302
6303         /* don't bother with service tasks if a reset is in progress */
6304         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
6305                 i40e_service_event_complete(pf);
6306                 return;
6307         }
6308
6309         i40e_reset_subtask(pf);
6310         i40e_handle_mdd_event(pf);
6311         i40e_vc_process_vflr_event(pf);
6312         i40e_watchdog_subtask(pf);
6313         i40e_fdir_reinit_subtask(pf);
6314         i40e_check_hang_subtask(pf);
6315         i40e_sync_filters_subtask(pf);
6316 #ifdef CONFIG_I40E_VXLAN
6317         i40e_sync_vxlan_filters_subtask(pf);
6318 #endif
6319         i40e_clean_adminq_subtask(pf);
6320
6321         i40e_link_event(pf);
6322
6323         i40e_service_event_complete(pf);
6324
6325         /* If the tasks have taken longer than one timer cycle or there
6326          * is more work to be done, reschedule the service task now
6327          * rather than wait for the timer to tick again.
6328          */
6329         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
6330             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
6331             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
6332             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
6333                 i40e_service_event_schedule(pf);
6334 }
6335
6336 /**
6337  * i40e_service_timer - timer callback
6338  * @data: pointer to PF struct
6339  **/
6340 static void i40e_service_timer(unsigned long data)
6341 {
6342         struct i40e_pf *pf = (struct i40e_pf *)data;
6343
6344         mod_timer(&pf->service_timer,
6345                   round_jiffies(jiffies + pf->service_timer_period));
6346         i40e_service_event_schedule(pf);
6347 }
6348
6349 /**
6350  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
6351  * @vsi: the VSI being configured
6352  **/
6353 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
6354 {
6355         struct i40e_pf *pf = vsi->back;
6356
6357         switch (vsi->type) {
6358         case I40E_VSI_MAIN:
6359                 vsi->alloc_queue_pairs = pf->num_lan_qps;
6360                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
6361                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
6362                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6363                         vsi->num_q_vectors = pf->num_lan_msix;
6364                 else
6365                         vsi->num_q_vectors = 1;
6366
6367                 break;
6368
6369         case I40E_VSI_FDIR:
6370                 vsi->alloc_queue_pairs = 1;
6371                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
6372                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
6373                 vsi->num_q_vectors = 1;
6374                 break;
6375
6376         case I40E_VSI_VMDQ2:
6377                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
6378                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
6379                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
6380                 vsi->num_q_vectors = pf->num_vmdq_msix;
6381                 break;
6382
6383         case I40E_VSI_SRIOV:
6384                 vsi->alloc_queue_pairs = pf->num_vf_qps;
6385                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
6386                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
6387                 break;
6388
6389 #ifdef I40E_FCOE
6390         case I40E_VSI_FCOE:
6391                 vsi->alloc_queue_pairs = pf->num_fcoe_qps;
6392                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
6393                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
6394                 vsi->num_q_vectors = pf->num_fcoe_msix;
6395                 break;
6396
6397 #endif /* I40E_FCOE */
6398         default:
6399                 WARN_ON(1);
6400                 return -ENODATA;
6401         }
6402
6403         return 0;
6404 }
6405
6406 /**
6407  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
6408  * @type: VSI pointer
6409  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
6410  *
6411  * On error: returns error code (negative)
6412  * On success: returns 0
6413  **/
6414 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
6415 {
6416         int size;
6417         int ret = 0;
6418
6419         /* allocate memory for both Tx and Rx ring pointers */
6420         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
6421         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
6422         if (!vsi->tx_rings)
6423                 return -ENOMEM;
6424         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
6425
6426         if (alloc_qvectors) {
6427                 /* allocate memory for q_vector pointers */
6428                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
6429                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
6430                 if (!vsi->q_vectors) {
6431                         ret = -ENOMEM;
6432                         goto err_vectors;
6433                 }
6434         }
6435         return ret;
6436
6437 err_vectors:
6438         kfree(vsi->tx_rings);
6439         return ret;
6440 }
6441
6442 /**
6443  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
6444  * @pf: board private structure
6445  * @type: type of VSI
6446  *
6447  * On error: returns error code (negative)
6448  * On success: returns vsi index in PF (positive)
6449  **/
6450 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
6451 {
6452         int ret = -ENODEV;
6453         struct i40e_vsi *vsi;
6454         int vsi_idx;
6455         int i;
6456
6457         /* Need to protect the allocation of the VSIs at the PF level */
6458         mutex_lock(&pf->switch_mutex);
6459
6460         /* VSI list may be fragmented if VSI creation/destruction has
6461          * been happening.  We can afford to do a quick scan to look
6462          * for any free VSIs in the list.
6463          *
6464          * find next empty vsi slot, looping back around if necessary
6465          */
6466         i = pf->next_vsi;
6467         while (i < pf->num_alloc_vsi && pf->vsi[i])
6468                 i++;
6469         if (i >= pf->num_alloc_vsi) {
6470                 i = 0;
6471                 while (i < pf->next_vsi && pf->vsi[i])
6472                         i++;
6473         }
6474
6475         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
6476                 vsi_idx = i;             /* Found one! */
6477         } else {
6478                 ret = -ENODEV;
6479                 goto unlock_pf;  /* out of VSI slots! */
6480         }
6481         pf->next_vsi = ++i;
6482
6483         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
6484         if (!vsi) {
6485                 ret = -ENOMEM;
6486                 goto unlock_pf;
6487         }
6488         vsi->type = type;
6489         vsi->back = pf;
6490         set_bit(__I40E_DOWN, &vsi->state);
6491         vsi->flags = 0;
6492         vsi->idx = vsi_idx;
6493         vsi->rx_itr_setting = pf->rx_itr_default;
6494         vsi->tx_itr_setting = pf->tx_itr_default;
6495         vsi->netdev_registered = false;
6496         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
6497         INIT_LIST_HEAD(&vsi->mac_filter_list);
6498         vsi->irqs_ready = false;
6499
6500         ret = i40e_set_num_rings_in_vsi(vsi);
6501         if (ret)
6502                 goto err_rings;
6503
6504         ret = i40e_vsi_alloc_arrays(vsi, true);
6505         if (ret)
6506                 goto err_rings;
6507
6508         /* Setup default MSIX irq handler for VSI */
6509         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
6510
6511         pf->vsi[vsi_idx] = vsi;
6512         ret = vsi_idx;
6513         goto unlock_pf;
6514
6515 err_rings:
6516         pf->next_vsi = i - 1;
6517         kfree(vsi);
6518 unlock_pf:
6519         mutex_unlock(&pf->switch_mutex);
6520         return ret;
6521 }
6522
6523 /**
6524  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
6525  * @type: VSI pointer
6526  * @free_qvectors: a bool to specify if q_vectors need to be freed.
6527  *
6528  * On error: returns error code (negative)
6529  * On success: returns 0
6530  **/
6531 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
6532 {
6533         /* free the ring and vector containers */
6534         if (free_qvectors) {
6535                 kfree(vsi->q_vectors);
6536                 vsi->q_vectors = NULL;
6537         }
6538         kfree(vsi->tx_rings);
6539         vsi->tx_rings = NULL;
6540         vsi->rx_rings = NULL;
6541 }
6542
6543 /**
6544  * i40e_vsi_clear - Deallocate the VSI provided
6545  * @vsi: the VSI being un-configured
6546  **/
6547 static int i40e_vsi_clear(struct i40e_vsi *vsi)
6548 {
6549         struct i40e_pf *pf;
6550
6551         if (!vsi)
6552                 return 0;
6553
6554         if (!vsi->back)
6555                 goto free_vsi;
6556         pf = vsi->back;
6557
6558         mutex_lock(&pf->switch_mutex);
6559         if (!pf->vsi[vsi->idx]) {
6560                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
6561                         vsi->idx, vsi->idx, vsi, vsi->type);
6562                 goto unlock_vsi;
6563         }
6564
6565         if (pf->vsi[vsi->idx] != vsi) {
6566                 dev_err(&pf->pdev->dev,
6567                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
6568                         pf->vsi[vsi->idx]->idx,
6569                         pf->vsi[vsi->idx],
6570                         pf->vsi[vsi->idx]->type,
6571                         vsi->idx, vsi, vsi->type);
6572                 goto unlock_vsi;
6573         }
6574
6575         /* updates the pf for this cleared vsi */
6576         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
6577         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
6578
6579         i40e_vsi_free_arrays(vsi, true);
6580
6581         pf->vsi[vsi->idx] = NULL;
6582         if (vsi->idx < pf->next_vsi)
6583                 pf->next_vsi = vsi->idx;
6584
6585 unlock_vsi:
6586         mutex_unlock(&pf->switch_mutex);
6587 free_vsi:
6588         kfree(vsi);
6589
6590         return 0;
6591 }
6592
6593 /**
6594  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
6595  * @vsi: the VSI being cleaned
6596  **/
6597 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
6598 {
6599         int i;
6600
6601         if (vsi->tx_rings && vsi->tx_rings[0]) {
6602                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
6603                         kfree_rcu(vsi->tx_rings[i], rcu);
6604                         vsi->tx_rings[i] = NULL;
6605                         vsi->rx_rings[i] = NULL;
6606                 }
6607         }
6608 }
6609
6610 /**
6611  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
6612  * @vsi: the VSI being configured
6613  **/
6614 static int i40e_alloc_rings(struct i40e_vsi *vsi)
6615 {
6616         struct i40e_ring *tx_ring, *rx_ring;
6617         struct i40e_pf *pf = vsi->back;
6618         int i;
6619
6620         /* Set basic values in the rings to be used later during open() */
6621         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
6622                 /* allocate space for both Tx and Rx in one shot */
6623                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
6624                 if (!tx_ring)
6625                         goto err_out;
6626
6627                 tx_ring->queue_index = i;
6628                 tx_ring->reg_idx = vsi->base_queue + i;
6629                 tx_ring->ring_active = false;
6630                 tx_ring->vsi = vsi;
6631                 tx_ring->netdev = vsi->netdev;
6632                 tx_ring->dev = &pf->pdev->dev;
6633                 tx_ring->count = vsi->num_desc;
6634                 tx_ring->size = 0;
6635                 tx_ring->dcb_tc = 0;
6636                 vsi->tx_rings[i] = tx_ring;
6637
6638                 rx_ring = &tx_ring[1];
6639                 rx_ring->queue_index = i;
6640                 rx_ring->reg_idx = vsi->base_queue + i;
6641                 rx_ring->ring_active = false;
6642                 rx_ring->vsi = vsi;
6643                 rx_ring->netdev = vsi->netdev;
6644                 rx_ring->dev = &pf->pdev->dev;
6645                 rx_ring->count = vsi->num_desc;
6646                 rx_ring->size = 0;
6647                 rx_ring->dcb_tc = 0;
6648                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
6649                         set_ring_16byte_desc_enabled(rx_ring);
6650                 else
6651                         clear_ring_16byte_desc_enabled(rx_ring);
6652                 vsi->rx_rings[i] = rx_ring;
6653         }
6654
6655         return 0;
6656
6657 err_out:
6658         i40e_vsi_clear_rings(vsi);
6659         return -ENOMEM;
6660 }
6661
6662 /**
6663  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
6664  * @pf: board private structure
6665  * @vectors: the number of MSI-X vectors to request
6666  *
6667  * Returns the number of vectors reserved, or error
6668  **/
6669 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
6670 {
6671         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
6672                                         I40E_MIN_MSIX, vectors);
6673         if (vectors < 0) {
6674                 dev_info(&pf->pdev->dev,
6675                          "MSI-X vector reservation failed: %d\n", vectors);
6676                 vectors = 0;
6677         }
6678
6679         return vectors;
6680 }
6681
6682 /**
6683  * i40e_init_msix - Setup the MSIX capability
6684  * @pf: board private structure
6685  *
6686  * Work with the OS to set up the MSIX vectors needed.
6687  *
6688  * Returns 0 on success, negative on failure
6689  **/
6690 static int i40e_init_msix(struct i40e_pf *pf)
6691 {
6692         i40e_status err = 0;
6693         struct i40e_hw *hw = &pf->hw;
6694         int v_budget, i;
6695         int vec;
6696
6697         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
6698                 return -ENODEV;
6699
6700         /* The number of vectors we'll request will be comprised of:
6701          *   - Add 1 for "other" cause for Admin Queue events, etc.
6702          *   - The number of LAN queue pairs
6703          *      - Queues being used for RSS.
6704          *              We don't need as many as max_rss_size vectors.
6705          *              use rss_size instead in the calculation since that
6706          *              is governed by number of cpus in the system.
6707          *      - assumes symmetric Tx/Rx pairing
6708          *   - The number of VMDq pairs
6709 #ifdef I40E_FCOE
6710          *   - The number of FCOE qps.
6711 #endif
6712          * Once we count this up, try the request.
6713          *
6714          * If we can't get what we want, we'll simplify to nearly nothing
6715          * and try again.  If that still fails, we punt.
6716          */
6717         pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
6718         pf->num_vmdq_msix = pf->num_vmdq_qps;
6719         v_budget = 1 + pf->num_lan_msix;
6720         v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
6721         if (pf->flags & I40E_FLAG_FD_SB_ENABLED)
6722                 v_budget++;
6723
6724 #ifdef I40E_FCOE
6725         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
6726                 pf->num_fcoe_msix = pf->num_fcoe_qps;
6727                 v_budget += pf->num_fcoe_msix;
6728         }
6729
6730 #endif
6731         /* Scale down if necessary, and the rings will share vectors */
6732         v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
6733
6734         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
6735                                    GFP_KERNEL);
6736         if (!pf->msix_entries)
6737                 return -ENOMEM;
6738
6739         for (i = 0; i < v_budget; i++)
6740                 pf->msix_entries[i].entry = i;
6741         vec = i40e_reserve_msix_vectors(pf, v_budget);
6742
6743         if (vec != v_budget) {
6744                 /* If we have limited resources, we will start with no vectors
6745                  * for the special features and then allocate vectors to some
6746                  * of these features based on the policy and at the end disable
6747                  * the features that did not get any vectors.
6748                  */
6749 #ifdef I40E_FCOE
6750                 pf->num_fcoe_qps = 0;
6751                 pf->num_fcoe_msix = 0;
6752 #endif
6753                 pf->num_vmdq_msix = 0;
6754         }
6755
6756         if (vec < I40E_MIN_MSIX) {
6757                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
6758                 kfree(pf->msix_entries);
6759                 pf->msix_entries = NULL;
6760                 return -ENODEV;
6761
6762         } else if (vec == I40E_MIN_MSIX) {
6763                 /* Adjust for minimal MSIX use */
6764                 pf->num_vmdq_vsis = 0;
6765                 pf->num_vmdq_qps = 0;
6766                 pf->num_lan_qps = 1;
6767                 pf->num_lan_msix = 1;
6768
6769         } else if (vec != v_budget) {
6770                 /* reserve the misc vector */
6771                 vec--;
6772
6773                 /* Scale vector usage down */
6774                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
6775                 pf->num_vmdq_vsis = 1;
6776
6777                 /* partition out the remaining vectors */
6778                 switch (vec) {
6779                 case 2:
6780                         pf->num_lan_msix = 1;
6781                         break;
6782                 case 3:
6783 #ifdef I40E_FCOE
6784                         /* give one vector to FCoE */
6785                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
6786                                 pf->num_lan_msix = 1;
6787                                 pf->num_fcoe_msix = 1;
6788                         }
6789 #else
6790                         pf->num_lan_msix = 2;
6791 #endif
6792                         break;
6793                 default:
6794 #ifdef I40E_FCOE
6795                         /* give one vector to FCoE */
6796                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
6797                                 pf->num_fcoe_msix = 1;
6798                                 vec--;
6799                         }
6800 #endif
6801                         pf->num_lan_msix = min_t(int, (vec / 2),
6802                                                  pf->num_lan_qps);
6803                         pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
6804                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
6805                         break;
6806                 }
6807         }
6808
6809         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
6810             (pf->num_vmdq_msix == 0)) {
6811                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
6812                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
6813         }
6814 #ifdef I40E_FCOE
6815
6816         if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) {
6817                 dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n");
6818                 pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
6819         }
6820 #endif
6821         return err;
6822 }
6823
6824 /**
6825  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
6826  * @vsi: the VSI being configured
6827  * @v_idx: index of the vector in the vsi struct
6828  *
6829  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
6830  **/
6831 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
6832 {
6833         struct i40e_q_vector *q_vector;
6834
6835         /* allocate q_vector */
6836         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
6837         if (!q_vector)
6838                 return -ENOMEM;
6839
6840         q_vector->vsi = vsi;
6841         q_vector->v_idx = v_idx;
6842         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
6843         if (vsi->netdev)
6844                 netif_napi_add(vsi->netdev, &q_vector->napi,
6845                                i40e_napi_poll, NAPI_POLL_WEIGHT);
6846
6847         q_vector->rx.latency_range = I40E_LOW_LATENCY;
6848         q_vector->tx.latency_range = I40E_LOW_LATENCY;
6849
6850         /* tie q_vector and vsi together */
6851         vsi->q_vectors[v_idx] = q_vector;
6852
6853         return 0;
6854 }
6855
6856 /**
6857  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
6858  * @vsi: the VSI being configured
6859  *
6860  * We allocate one q_vector per queue interrupt.  If allocation fails we
6861  * return -ENOMEM.
6862  **/
6863 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
6864 {
6865         struct i40e_pf *pf = vsi->back;
6866         int v_idx, num_q_vectors;
6867         int err;
6868
6869         /* if not MSIX, give the one vector only to the LAN VSI */
6870         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6871                 num_q_vectors = vsi->num_q_vectors;
6872         else if (vsi == pf->vsi[pf->lan_vsi])
6873                 num_q_vectors = 1;
6874         else
6875                 return -EINVAL;
6876
6877         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
6878                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
6879                 if (err)
6880                         goto err_out;
6881         }
6882
6883         return 0;
6884
6885 err_out:
6886         while (v_idx--)
6887                 i40e_free_q_vector(vsi, v_idx);
6888
6889         return err;
6890 }
6891
6892 /**
6893  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
6894  * @pf: board private structure to initialize
6895  **/
6896 static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
6897 {
6898         int err = 0;
6899
6900         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
6901                 err = i40e_init_msix(pf);
6902                 if (err) {
6903                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
6904 #ifdef I40E_FCOE
6905                                        I40E_FLAG_FCOE_ENABLED   |
6906 #endif
6907                                        I40E_FLAG_RSS_ENABLED    |
6908                                        I40E_FLAG_DCB_CAPABLE    |
6909                                        I40E_FLAG_SRIOV_ENABLED  |
6910                                        I40E_FLAG_FD_SB_ENABLED  |
6911                                        I40E_FLAG_FD_ATR_ENABLED |
6912                                        I40E_FLAG_VMDQ_ENABLED);
6913
6914                         /* rework the queue expectations without MSIX */
6915                         i40e_determine_queue_usage(pf);
6916                 }
6917         }
6918
6919         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
6920             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
6921                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
6922                 err = pci_enable_msi(pf->pdev);
6923                 if (err) {
6924                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
6925                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
6926                 }
6927         }
6928
6929         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
6930                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
6931
6932         /* track first vector for misc interrupts */
6933         err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
6934 }
6935
6936 /**
6937  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
6938  * @pf: board private structure
6939  *
6940  * This sets up the handler for MSIX 0, which is used to manage the
6941  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
6942  * when in MSI or Legacy interrupt mode.
6943  **/
6944 static int i40e_setup_misc_vector(struct i40e_pf *pf)
6945 {
6946         struct i40e_hw *hw = &pf->hw;
6947         int err = 0;
6948
6949         /* Only request the irq if this is the first time through, and
6950          * not when we're rebuilding after a Reset
6951          */
6952         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
6953                 err = request_irq(pf->msix_entries[0].vector,
6954                                   i40e_intr, 0, pf->misc_int_name, pf);
6955                 if (err) {
6956                         dev_info(&pf->pdev->dev,
6957                                  "request_irq for %s failed: %d\n",
6958                                  pf->misc_int_name, err);
6959                         return -EFAULT;
6960                 }
6961         }
6962
6963         i40e_enable_misc_int_causes(hw);
6964
6965         /* associate no queues to the misc vector */
6966         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
6967         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
6968
6969         i40e_flush(hw);
6970
6971         i40e_irq_dynamic_enable_icr0(pf);
6972
6973         return err;
6974 }
6975
6976 /**
6977  * i40e_config_rss - Prepare for RSS if used
6978  * @pf: board private structure
6979  **/
6980 static int i40e_config_rss(struct i40e_pf *pf)
6981 {
6982         /* Set of random keys generated using kernel random number generator */
6983         static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
6984                                 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
6985                                 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
6986                                 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
6987         struct i40e_hw *hw = &pf->hw;
6988         u32 lut = 0;
6989         int i, j;
6990         u64 hena;
6991         u32 reg_val;
6992
6993         /* Fill out hash function seed */
6994         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6995                 wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
6996
6997         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
6998         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
6999                 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
7000         hena |= I40E_DEFAULT_RSS_HENA;
7001         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
7002         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
7003
7004         /* Check capability and Set table size and register per hw expectation*/
7005         reg_val = rd32(hw, I40E_PFQF_CTL_0);
7006         if (hw->func_caps.rss_table_size == 512) {
7007                 reg_val |= I40E_PFQF_CTL_0_HASHLUTSIZE_512;
7008                 pf->rss_table_size = 512;
7009         } else {
7010                 pf->rss_table_size = 128;
7011                 reg_val &= ~I40E_PFQF_CTL_0_HASHLUTSIZE_512;
7012         }
7013         wr32(hw, I40E_PFQF_CTL_0, reg_val);
7014
7015         /* Populate the LUT with max no. of queues in round robin fashion */
7016         for (i = 0, j = 0; i < pf->rss_table_size; i++, j++) {
7017
7018                 /* The assumption is that lan qp count will be the highest
7019                  * qp count for any PF VSI that needs RSS.
7020                  * If multiple VSIs need RSS support, all the qp counts
7021                  * for those VSIs should be a power of 2 for RSS to work.
7022                  * If LAN VSI is the only consumer for RSS then this requirement
7023                  * is not necessary.
7024                  */
7025                 if (j == pf->rss_size)
7026                         j = 0;
7027                 /* lut = 4-byte sliding window of 4 lut entries */
7028                 lut = (lut << 8) | (j &
7029                          ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
7030                 /* On i = 3, we have 4 entries in lut; write to the register */
7031                 if ((i & 3) == 3)
7032                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
7033         }
7034         i40e_flush(hw);
7035
7036         return 0;
7037 }
7038
7039 /**
7040  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
7041  * @pf: board private structure
7042  * @queue_count: the requested queue count for rss.
7043  *
7044  * returns 0 if rss is not enabled, if enabled returns the final rss queue
7045  * count which may be different from the requested queue count.
7046  **/
7047 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
7048 {
7049         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
7050                 return 0;
7051
7052         queue_count = min_t(int, queue_count, pf->rss_size_max);
7053
7054         if (queue_count != pf->rss_size) {
7055                 i40e_prep_for_reset(pf);
7056
7057                 pf->rss_size = queue_count;
7058
7059                 i40e_reset_and_rebuild(pf, true);
7060                 i40e_config_rss(pf);
7061         }
7062         dev_info(&pf->pdev->dev, "RSS count:  %d\n", pf->rss_size);
7063         return pf->rss_size;
7064 }
7065
7066 /**
7067  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
7068  * @pf: board private structure to initialize
7069  *
7070  * i40e_sw_init initializes the Adapter private data structure.
7071  * Fields are initialized based on PCI device information and
7072  * OS network device settings (MTU size).
7073  **/
7074 static int i40e_sw_init(struct i40e_pf *pf)
7075 {
7076         int err = 0;
7077         int size;
7078
7079         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
7080                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
7081         pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
7082         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
7083                 if (I40E_DEBUG_USER & debug)
7084                         pf->hw.debug_mask = debug;
7085                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
7086                                                 I40E_DEFAULT_MSG_ENABLE);
7087         }
7088
7089         /* Set default capability flags */
7090         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
7091                     I40E_FLAG_MSI_ENABLED     |
7092                     I40E_FLAG_MSIX_ENABLED    |
7093                     I40E_FLAG_RX_1BUF_ENABLED;
7094
7095         /* Set default ITR */
7096         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
7097         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
7098
7099         /* Depending on PF configurations, it is possible that the RSS
7100          * maximum might end up larger than the available queues
7101          */
7102         pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
7103         pf->rss_size = 1;
7104         pf->rss_size_max = min_t(int, pf->rss_size_max,
7105                                  pf->hw.func_caps.num_tx_qp);
7106         if (pf->hw.func_caps.rss) {
7107                 pf->flags |= I40E_FLAG_RSS_ENABLED;
7108                 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
7109         }
7110
7111         /* MFP mode enabled */
7112         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
7113                 pf->flags |= I40E_FLAG_MFP_ENABLED;
7114                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
7115         }
7116
7117         /* FW/NVM is not yet fixed in this regard */
7118         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
7119             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
7120                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
7121                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
7122                 /* Setup a counter for fd_atr per pf */
7123                 pf->fd_atr_cnt_idx = I40E_FD_ATR_STAT_IDX(pf->hw.pf_id);
7124                 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) {
7125                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7126                         /* Setup a counter for fd_sb per pf */
7127                         pf->fd_sb_cnt_idx = I40E_FD_SB_STAT_IDX(pf->hw.pf_id);
7128                 } else {
7129                         dev_info(&pf->pdev->dev,
7130                                  "Flow Director Sideband mode Disabled in MFP mode\n");
7131                 }
7132                 pf->fdir_pf_filter_count =
7133                                  pf->hw.func_caps.fd_filters_guaranteed;
7134                 pf->hw.fdir_shared_filter_count =
7135                                  pf->hw.func_caps.fd_filters_best_effort;
7136         }
7137
7138         if (pf->hw.func_caps.vmdq) {
7139                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
7140                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
7141                 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
7142         }
7143
7144 #ifdef I40E_FCOE
7145         err = i40e_init_pf_fcoe(pf);
7146         if (err)
7147                 dev_info(&pf->pdev->dev, "init_pf_fcoe failed: %d\n", err);
7148
7149 #endif /* I40E_FCOE */
7150 #ifdef CONFIG_PCI_IOV
7151         if (pf->hw.func_caps.num_vfs) {
7152                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
7153                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
7154                 pf->num_req_vfs = min_t(int,
7155                                         pf->hw.func_caps.num_vfs,
7156                                         I40E_MAX_VF_COUNT);
7157         }
7158 #endif /* CONFIG_PCI_IOV */
7159         pf->eeprom_version = 0xDEAD;
7160         pf->lan_veb = I40E_NO_VEB;
7161         pf->lan_vsi = I40E_NO_VSI;
7162
7163         /* set up queue assignment tracking */
7164         size = sizeof(struct i40e_lump_tracking)
7165                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
7166         pf->qp_pile = kzalloc(size, GFP_KERNEL);
7167         if (!pf->qp_pile) {
7168                 err = -ENOMEM;
7169                 goto sw_init_done;
7170         }
7171         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
7172         pf->qp_pile->search_hint = 0;
7173
7174         /* set up vector assignment tracking */
7175         size = sizeof(struct i40e_lump_tracking)
7176                 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
7177         pf->irq_pile = kzalloc(size, GFP_KERNEL);
7178         if (!pf->irq_pile) {
7179                 kfree(pf->qp_pile);
7180                 err = -ENOMEM;
7181                 goto sw_init_done;
7182         }
7183         pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
7184         pf->irq_pile->search_hint = 0;
7185
7186         pf->tx_timeout_recovery_level = 1;
7187
7188         mutex_init(&pf->switch_mutex);
7189
7190 sw_init_done:
7191         return err;
7192 }
7193
7194 /**
7195  * i40e_set_ntuple - set the ntuple feature flag and take action
7196  * @pf: board private structure to initialize
7197  * @features: the feature set that the stack is suggesting
7198  *
7199  * returns a bool to indicate if reset needs to happen
7200  **/
7201 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
7202 {
7203         bool need_reset = false;
7204
7205         /* Check if Flow Director n-tuple support was enabled or disabled.  If
7206          * the state changed, we need to reset.
7207          */
7208         if (features & NETIF_F_NTUPLE) {
7209                 /* Enable filters and mark for reset */
7210                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
7211                         need_reset = true;
7212                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7213         } else {
7214                 /* turn off filters, mark for reset and clear SW filter list */
7215                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7216                         need_reset = true;
7217                         i40e_fdir_filter_exit(pf);
7218                 }
7219                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7220                 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
7221                 /* reset fd counters */
7222                 pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0;
7223                 pf->fdir_pf_active_filters = 0;
7224                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
7225                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
7226                 /* if ATR was auto disabled it can be re-enabled. */
7227                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
7228                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
7229                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
7230         }
7231         return need_reset;
7232 }
7233
7234 /**
7235  * i40e_set_features - set the netdev feature flags
7236  * @netdev: ptr to the netdev being adjusted
7237  * @features: the feature set that the stack is suggesting
7238  **/
7239 static int i40e_set_features(struct net_device *netdev,
7240                              netdev_features_t features)
7241 {
7242         struct i40e_netdev_priv *np = netdev_priv(netdev);
7243         struct i40e_vsi *vsi = np->vsi;
7244         struct i40e_pf *pf = vsi->back;
7245         bool need_reset;
7246
7247         if (features & NETIF_F_HW_VLAN_CTAG_RX)
7248                 i40e_vlan_stripping_enable(vsi);
7249         else
7250                 i40e_vlan_stripping_disable(vsi);
7251
7252         need_reset = i40e_set_ntuple(pf, features);
7253
7254         if (need_reset)
7255                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
7256
7257         return 0;
7258 }
7259
7260 #ifdef CONFIG_I40E_VXLAN
7261 /**
7262  * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
7263  * @pf: board private structure
7264  * @port: The UDP port to look up
7265  *
7266  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
7267  **/
7268 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
7269 {
7270         u8 i;
7271
7272         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7273                 if (pf->vxlan_ports[i] == port)
7274                         return i;
7275         }
7276
7277         return i;
7278 }
7279
7280 /**
7281  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
7282  * @netdev: This physical port's netdev
7283  * @sa_family: Socket Family that VXLAN is notifying us about
7284  * @port: New UDP port number that VXLAN started listening to
7285  **/
7286 static void i40e_add_vxlan_port(struct net_device *netdev,
7287                                 sa_family_t sa_family, __be16 port)
7288 {
7289         struct i40e_netdev_priv *np = netdev_priv(netdev);
7290         struct i40e_vsi *vsi = np->vsi;
7291         struct i40e_pf *pf = vsi->back;
7292         u8 next_idx;
7293         u8 idx;
7294
7295         if (sa_family == AF_INET6)
7296                 return;
7297
7298         idx = i40e_get_vxlan_port_idx(pf, port);
7299
7300         /* Check if port already exists */
7301         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
7302                 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
7303                 return;
7304         }
7305
7306         /* Now check if there is space to add the new port */
7307         next_idx = i40e_get_vxlan_port_idx(pf, 0);
7308
7309         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
7310                 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
7311                             ntohs(port));
7312                 return;
7313         }
7314
7315         /* New port: add it and mark its index in the bitmap */
7316         pf->vxlan_ports[next_idx] = port;
7317         pf->pending_vxlan_bitmap |= (1 << next_idx);
7318
7319         pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
7320 }
7321
7322 /**
7323  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
7324  * @netdev: This physical port's netdev
7325  * @sa_family: Socket Family that VXLAN is notifying us about
7326  * @port: UDP port number that VXLAN stopped listening to
7327  **/
7328 static void i40e_del_vxlan_port(struct net_device *netdev,
7329                                 sa_family_t sa_family, __be16 port)
7330 {
7331         struct i40e_netdev_priv *np = netdev_priv(netdev);
7332         struct i40e_vsi *vsi = np->vsi;
7333         struct i40e_pf *pf = vsi->back;
7334         u8 idx;
7335
7336         if (sa_family == AF_INET6)
7337                 return;
7338
7339         idx = i40e_get_vxlan_port_idx(pf, port);
7340
7341         /* Check if port already exists */
7342         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
7343                 /* if port exists, set it to 0 (mark for deletion)
7344                  * and make it pending
7345                  */
7346                 pf->vxlan_ports[idx] = 0;
7347
7348                 pf->pending_vxlan_bitmap |= (1 << idx);
7349
7350                 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
7351         } else {
7352                 netdev_warn(netdev, "Port %d was not found, not deleting\n",
7353                             ntohs(port));
7354         }
7355 }
7356
7357 #endif
7358 static int i40e_get_phys_port_id(struct net_device *netdev,
7359                                  struct netdev_phys_port_id *ppid)
7360 {
7361         struct i40e_netdev_priv *np = netdev_priv(netdev);
7362         struct i40e_pf *pf = np->vsi->back;
7363         struct i40e_hw *hw = &pf->hw;
7364
7365         if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
7366                 return -EOPNOTSUPP;
7367
7368         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
7369         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
7370
7371         return 0;
7372 }
7373
7374 #ifdef HAVE_FDB_OPS
7375 #ifdef USE_CONST_DEV_UC_CHAR
7376 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
7377                             struct net_device *dev,
7378                             const unsigned char *addr,
7379                             u16 flags)
7380 #else
7381 static int i40e_ndo_fdb_add(struct ndmsg *ndm,
7382                             struct net_device *dev,
7383                             unsigned char *addr,
7384                             u16 flags)
7385 #endif
7386 {
7387         struct i40e_netdev_priv *np = netdev_priv(dev);
7388         struct i40e_pf *pf = np->vsi->back;
7389         int err = 0;
7390
7391         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
7392                 return -EOPNOTSUPP;
7393
7394         /* Hardware does not support aging addresses so if a
7395          * ndm_state is given only allow permanent addresses
7396          */
7397         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
7398                 netdev_info(dev, "FDB only supports static addresses\n");
7399                 return -EINVAL;
7400         }
7401
7402         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
7403                 err = dev_uc_add_excl(dev, addr);
7404         else if (is_multicast_ether_addr(addr))
7405                 err = dev_mc_add_excl(dev, addr);
7406         else
7407                 err = -EINVAL;
7408
7409         /* Only return duplicate errors if NLM_F_EXCL is set */
7410         if (err == -EEXIST && !(flags & NLM_F_EXCL))
7411                 err = 0;
7412
7413         return err;
7414 }
7415
7416 #ifndef USE_DEFAULT_FDB_DEL_DUMP
7417 #ifdef USE_CONST_DEV_UC_CHAR
7418 static int i40e_ndo_fdb_del(struct ndmsg *ndm,
7419                             struct net_device *dev,
7420                             const unsigned char *addr)
7421 #else
7422 static int i40e_ndo_fdb_del(struct ndmsg *ndm,
7423                             struct net_device *dev,
7424                             unsigned char *addr)
7425 #endif
7426 {
7427         struct i40e_netdev_priv *np = netdev_priv(dev);
7428         struct i40e_pf *pf = np->vsi->back;
7429         int err = -EOPNOTSUPP;
7430
7431         if (ndm->ndm_state & NUD_PERMANENT) {
7432                 netdev_info(dev, "FDB only supports static addresses\n");
7433                 return -EINVAL;
7434         }
7435
7436         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
7437                 if (is_unicast_ether_addr(addr))
7438                         err = dev_uc_del(dev, addr);
7439                 else if (is_multicast_ether_addr(addr))
7440                         err = dev_mc_del(dev, addr);
7441                 else
7442                         err = -EINVAL;
7443         }
7444
7445         return err;
7446 }
7447
7448 static int i40e_ndo_fdb_dump(struct sk_buff *skb,
7449                              struct netlink_callback *cb,
7450                              struct net_device *dev,
7451                              struct net_device *filter_dev,
7452                              int idx)
7453 {
7454         struct i40e_netdev_priv *np = netdev_priv(dev);
7455         struct i40e_pf *pf = np->vsi->back;
7456
7457         if (pf->flags & I40E_FLAG_SRIOV_ENABLED)
7458                 idx = ndo_dflt_fdb_dump(skb, cb, dev, filter_dev, idx);
7459
7460         return idx;
7461 }
7462
7463 #endif /* USE_DEFAULT_FDB_DEL_DUMP */
7464 #endif /* HAVE_FDB_OPS */
7465 static const struct net_device_ops i40e_netdev_ops = {
7466         .ndo_open               = i40e_open,
7467         .ndo_stop               = i40e_close,
7468         .ndo_start_xmit         = i40e_lan_xmit_frame,
7469         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
7470         .ndo_set_rx_mode        = i40e_set_rx_mode,
7471         .ndo_validate_addr      = eth_validate_addr,
7472         .ndo_set_mac_address    = i40e_set_mac,
7473         .ndo_change_mtu         = i40e_change_mtu,
7474         .ndo_do_ioctl           = i40e_ioctl,
7475         .ndo_tx_timeout         = i40e_tx_timeout,
7476         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
7477         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
7478 #ifdef CONFIG_NET_POLL_CONTROLLER
7479         .ndo_poll_controller    = i40e_netpoll,
7480 #endif
7481         .ndo_setup_tc           = i40e_setup_tc,
7482 #ifdef I40E_FCOE
7483         .ndo_fcoe_enable        = i40e_fcoe_enable,
7484         .ndo_fcoe_disable       = i40e_fcoe_disable,
7485 #endif
7486         .ndo_set_features       = i40e_set_features,
7487         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
7488         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
7489         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
7490         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
7491         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
7492         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
7493 #ifdef CONFIG_I40E_VXLAN
7494         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
7495         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
7496 #endif
7497         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
7498 #ifdef HAVE_FDB_OPS
7499         .ndo_fdb_add            = i40e_ndo_fdb_add,
7500 #ifndef USE_DEFAULT_FDB_DEL_DUMP
7501         .ndo_fdb_del            = i40e_ndo_fdb_del,
7502         .ndo_fdb_dump           = i40e_ndo_fdb_dump,
7503 #endif
7504 #endif
7505 };
7506
7507 /**
7508  * i40e_config_netdev - Setup the netdev flags
7509  * @vsi: the VSI being configured
7510  *
7511  * Returns 0 on success, negative value on failure
7512  **/
7513 static int i40e_config_netdev(struct i40e_vsi *vsi)
7514 {
7515         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
7516         struct i40e_pf *pf = vsi->back;
7517         struct i40e_hw *hw = &pf->hw;
7518         struct i40e_netdev_priv *np;
7519         struct net_device *netdev;
7520         u8 mac_addr[ETH_ALEN];
7521         int etherdev_size;
7522
7523         etherdev_size = sizeof(struct i40e_netdev_priv);
7524         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
7525         if (!netdev)
7526                 return -ENOMEM;
7527
7528         vsi->netdev = netdev;
7529         np = netdev_priv(netdev);
7530         np->vsi = vsi;
7531
7532         netdev->hw_enc_features |= NETIF_F_IP_CSUM       |
7533                                   NETIF_F_GSO_UDP_TUNNEL |
7534                                   NETIF_F_TSO;
7535
7536         netdev->features = NETIF_F_SG                  |
7537                            NETIF_F_IP_CSUM             |
7538                            NETIF_F_SCTP_CSUM           |
7539                            NETIF_F_HIGHDMA             |
7540                            NETIF_F_GSO_UDP_TUNNEL      |
7541                            NETIF_F_HW_VLAN_CTAG_TX     |
7542                            NETIF_F_HW_VLAN_CTAG_RX     |
7543                            NETIF_F_HW_VLAN_CTAG_FILTER |
7544                            NETIF_F_IPV6_CSUM           |
7545                            NETIF_F_TSO                 |
7546                            NETIF_F_TSO_ECN             |
7547                            NETIF_F_TSO6                |
7548                            NETIF_F_RXCSUM              |
7549                            NETIF_F_RXHASH              |
7550                            0;
7551
7552         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
7553                 netdev->features |= NETIF_F_NTUPLE;
7554
7555         /* copy netdev features into list of user selectable features */
7556         netdev->hw_features |= netdev->features;
7557
7558         if (vsi->type == I40E_VSI_MAIN) {
7559                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
7560                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
7561                 /* The following steps are necessary to prevent reception
7562                  * of tagged packets - some older NVM configurations load a
7563                  * default a MAC-VLAN filter that accepts any tagged packet
7564                  * which must be replaced by a normal filter.
7565                  */
7566                 if (!i40e_rm_default_mac_filter(vsi, mac_addr))
7567                         i40e_add_filter(vsi, mac_addr,
7568                                         I40E_VLAN_ANY, false, true);
7569         } else {
7570                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
7571                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
7572                          pf->vsi[pf->lan_vsi]->netdev->name);
7573                 random_ether_addr(mac_addr);
7574                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
7575         }
7576         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
7577
7578         ether_addr_copy(netdev->dev_addr, mac_addr);
7579         ether_addr_copy(netdev->perm_addr, mac_addr);
7580         /* vlan gets same features (except vlan offload)
7581          * after any tweaks for specific VSI types
7582          */
7583         netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
7584                                                      NETIF_F_HW_VLAN_CTAG_RX |
7585                                                    NETIF_F_HW_VLAN_CTAG_FILTER);
7586         netdev->priv_flags |= IFF_UNICAST_FLT;
7587         netdev->priv_flags |= IFF_SUPP_NOFCS;
7588         /* Setup netdev TC information */
7589         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
7590
7591         netdev->netdev_ops = &i40e_netdev_ops;
7592         netdev->watchdog_timeo = 5 * HZ;
7593         i40e_set_ethtool_ops(netdev);
7594 #ifdef I40E_FCOE
7595         i40e_fcoe_config_netdev(netdev, vsi);
7596 #endif
7597
7598         return 0;
7599 }
7600
7601 /**
7602  * i40e_vsi_delete - Delete a VSI from the switch
7603  * @vsi: the VSI being removed
7604  *
7605  * Returns 0 on success, negative value on failure
7606  **/
7607 static void i40e_vsi_delete(struct i40e_vsi *vsi)
7608 {
7609         /* remove default VSI is not allowed */
7610         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
7611                 return;
7612
7613         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
7614 }
7615
7616 /**
7617  * i40e_add_vsi - Add a VSI to the switch
7618  * @vsi: the VSI being configured
7619  *
7620  * This initializes a VSI context depending on the VSI type to be added and
7621  * passes it down to the add_vsi aq command.
7622  **/
7623 static int i40e_add_vsi(struct i40e_vsi *vsi)
7624 {
7625         int ret = -ENODEV;
7626         struct i40e_mac_filter *f, *ftmp;
7627         struct i40e_pf *pf = vsi->back;
7628         struct i40e_hw *hw = &pf->hw;
7629         struct i40e_vsi_context ctxt;
7630         u8 enabled_tc = 0x1; /* TC0 enabled */
7631         int f_count = 0;
7632
7633         memset(&ctxt, 0, sizeof(ctxt));
7634         switch (vsi->type) {
7635         case I40E_VSI_MAIN:
7636                 /* The PF's main VSI is already setup as part of the
7637                  * device initialization, so we'll not bother with
7638                  * the add_vsi call, but we will retrieve the current
7639                  * VSI context.
7640                  */
7641                 ctxt.seid = pf->main_vsi_seid;
7642                 ctxt.pf_num = pf->hw.pf_id;
7643                 ctxt.vf_num = 0;
7644                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
7645                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
7646                 if (ret) {
7647                         dev_info(&pf->pdev->dev,
7648                                  "couldn't get pf vsi config, err %d, aq_err %d\n",
7649                                  ret, pf->hw.aq.asq_last_status);
7650                         return -ENOENT;
7651                 }
7652                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
7653                 vsi->info.valid_sections = 0;
7654
7655                 vsi->seid = ctxt.seid;
7656                 vsi->id = ctxt.vsi_number;
7657
7658                 enabled_tc = i40e_pf_get_tc_map(pf);
7659
7660                 /* MFP mode setup queue map and update VSI */
7661                 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7662                         memset(&ctxt, 0, sizeof(ctxt));
7663                         ctxt.seid = pf->main_vsi_seid;
7664                         ctxt.pf_num = pf->hw.pf_id;
7665                         ctxt.vf_num = 0;
7666                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
7667                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7668                         if (ret) {
7669                                 dev_info(&pf->pdev->dev,
7670                                          "update vsi failed, aq_err=%d\n",
7671                                          pf->hw.aq.asq_last_status);
7672                                 ret = -ENOENT;
7673                                 goto err;
7674                         }
7675                         /* update the local VSI info queue map */
7676                         i40e_vsi_update_queue_map(vsi, &ctxt);
7677                         vsi->info.valid_sections = 0;
7678                 } else {
7679                         /* Default/Main VSI is only enabled for TC0
7680                          * reconfigure it to enable all TCs that are
7681                          * available on the port in SFP mode.
7682                          */
7683                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7684                         if (ret) {
7685                                 dev_info(&pf->pdev->dev,
7686                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
7687                                          enabled_tc, ret,
7688                                          pf->hw.aq.asq_last_status);
7689                                 ret = -ENOENT;
7690                         }
7691                 }
7692                 break;
7693
7694         case I40E_VSI_FDIR:
7695                 ctxt.pf_num = hw->pf_id;
7696                 ctxt.vf_num = 0;
7697                 ctxt.uplink_seid = vsi->uplink_seid;
7698                 ctxt.connection_type = 0x1;     /* regular data port */
7699                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
7700                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7701                 break;
7702
7703         case I40E_VSI_VMDQ2:
7704                 ctxt.pf_num = hw->pf_id;
7705                 ctxt.vf_num = 0;
7706                 ctxt.uplink_seid = vsi->uplink_seid;
7707                 ctxt.connection_type = 0x1;     /* regular data port */
7708                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
7709
7710                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
7711
7712                 /* This VSI is connected to VEB so the switch_id
7713                  * should be set to zero by default.
7714                  */
7715                 ctxt.info.switch_id = 0;
7716                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
7717
7718                 /* Setup the VSI tx/rx queue map for TC0 only for now */
7719                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7720                 break;
7721
7722         case I40E_VSI_SRIOV:
7723                 ctxt.pf_num = hw->pf_id;
7724                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
7725                 ctxt.uplink_seid = vsi->uplink_seid;
7726                 ctxt.connection_type = 0x1;     /* regular data port */
7727                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
7728
7729                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
7730
7731                 /* This VSI is connected to VEB so the switch_id
7732                  * should be set to zero by default.
7733                  */
7734                 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
7735
7736                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
7737                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
7738                 if (pf->vf[vsi->vf_id].spoofchk) {
7739                         ctxt.info.valid_sections |=
7740                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
7741                         ctxt.info.sec_flags |=
7742                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
7743                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
7744                 }
7745                 /* Setup the VSI tx/rx queue map for TC0 only for now */
7746                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7747                 break;
7748
7749 #ifdef I40E_FCOE
7750         case I40E_VSI_FCOE:
7751                 ret = i40e_fcoe_vsi_init(vsi, &ctxt);
7752                 if (ret) {
7753                         dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n");
7754                         return ret;
7755                 }
7756                 break;
7757
7758 #endif /* I40E_FCOE */
7759         default:
7760                 return -ENODEV;
7761         }
7762
7763         if (vsi->type != I40E_VSI_MAIN) {
7764                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
7765                 if (ret) {
7766                         dev_info(&vsi->back->pdev->dev,
7767                                  "add vsi failed, aq_err=%d\n",
7768                                  vsi->back->hw.aq.asq_last_status);
7769                         ret = -ENOENT;
7770                         goto err;
7771                 }
7772                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
7773                 vsi->info.valid_sections = 0;
7774                 vsi->seid = ctxt.seid;
7775                 vsi->id = ctxt.vsi_number;
7776         }
7777
7778         /* If macvlan filters already exist, force them to get loaded */
7779         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
7780                 f->changed = true;
7781                 f_count++;
7782
7783                 if (f->is_laa && vsi->type == I40E_VSI_MAIN) {
7784                         struct i40e_aqc_remove_macvlan_element_data element;
7785
7786                         memset(&element, 0, sizeof(element));
7787                         ether_addr_copy(element.mac_addr, f->macaddr);
7788                         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7789                         ret = i40e_aq_remove_macvlan(hw, vsi->seid,
7790                                                      &element, 1, NULL);
7791                         if (ret) {
7792                                 /* some older FW has a different default */
7793                                 element.flags |=
7794                                                I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
7795                                 i40e_aq_remove_macvlan(hw, vsi->seid,
7796                                                        &element, 1, NULL);
7797                         }
7798
7799                         i40e_aq_mac_address_write(hw,
7800                                                   I40E_AQC_WRITE_TYPE_LAA_WOL,
7801                                                   f->macaddr, NULL);
7802                 }
7803         }
7804         if (f_count) {
7805                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
7806                 pf->flags |= I40E_FLAG_FILTER_SYNC;
7807         }
7808
7809         /* Update VSI BW information */
7810         ret = i40e_vsi_get_bw_info(vsi);
7811         if (ret) {
7812                 dev_info(&pf->pdev->dev,
7813                          "couldn't get vsi bw info, err %d, aq_err %d\n",
7814                          ret, pf->hw.aq.asq_last_status);
7815                 /* VSI is already added so not tearing that up */
7816                 ret = 0;
7817         }
7818
7819 err:
7820         return ret;
7821 }
7822
7823 /**
7824  * i40e_vsi_release - Delete a VSI and free its resources
7825  * @vsi: the VSI being removed
7826  *
7827  * Returns 0 on success or < 0 on error
7828  **/
7829 int i40e_vsi_release(struct i40e_vsi *vsi)
7830 {
7831         struct i40e_mac_filter *f, *ftmp;
7832         struct i40e_veb *veb = NULL;
7833         struct i40e_pf *pf;
7834         u16 uplink_seid;
7835         int i, n;
7836
7837         pf = vsi->back;
7838
7839         /* release of a VEB-owner or last VSI is not allowed */
7840         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
7841                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
7842                          vsi->seid, vsi->uplink_seid);
7843                 return -ENODEV;
7844         }
7845         if (vsi == pf->vsi[pf->lan_vsi] &&
7846             !test_bit(__I40E_DOWN, &pf->state)) {
7847                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
7848                 return -ENODEV;
7849         }
7850
7851         uplink_seid = vsi->uplink_seid;
7852         if (vsi->type != I40E_VSI_SRIOV) {
7853                 if (vsi->netdev_registered) {
7854                         vsi->netdev_registered = false;
7855                         if (vsi->netdev) {
7856                                 /* results in a call to i40e_close() */
7857                                 unregister_netdev(vsi->netdev);
7858                         }
7859                 } else {
7860                         i40e_vsi_close(vsi);
7861                 }
7862                 i40e_vsi_disable_irq(vsi);
7863         }
7864
7865         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
7866                 i40e_del_filter(vsi, f->macaddr, f->vlan,
7867                                 f->is_vf, f->is_netdev);
7868         i40e_sync_vsi_filters(vsi);
7869
7870         i40e_vsi_delete(vsi);
7871         i40e_vsi_free_q_vectors(vsi);
7872         if (vsi->netdev) {
7873                 free_netdev(vsi->netdev);
7874                 vsi->netdev = NULL;
7875         }
7876         i40e_vsi_clear_rings(vsi);
7877         i40e_vsi_clear(vsi);
7878
7879         /* If this was the last thing on the VEB, except for the
7880          * controlling VSI, remove the VEB, which puts the controlling
7881          * VSI onto the next level down in the switch.
7882          *
7883          * Well, okay, there's one more exception here: don't remove
7884          * the orphan VEBs yet.  We'll wait for an explicit remove request
7885          * from up the network stack.
7886          */
7887         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
7888                 if (pf->vsi[i] &&
7889                     pf->vsi[i]->uplink_seid == uplink_seid &&
7890                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
7891                         n++;      /* count the VSIs */
7892                 }
7893         }
7894         for (i = 0; i < I40E_MAX_VEB; i++) {
7895                 if (!pf->veb[i])
7896                         continue;
7897                 if (pf->veb[i]->uplink_seid == uplink_seid)
7898                         n++;     /* count the VEBs */
7899                 if (pf->veb[i]->seid == uplink_seid)
7900                         veb = pf->veb[i];
7901         }
7902         if (n == 0 && veb && veb->uplink_seid != 0)
7903                 i40e_veb_release(veb);
7904
7905         return 0;
7906 }
7907
7908 /**
7909  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
7910  * @vsi: ptr to the VSI
7911  *
7912  * This should only be called after i40e_vsi_mem_alloc() which allocates the
7913  * corresponding SW VSI structure and initializes num_queue_pairs for the
7914  * newly allocated VSI.
7915  *
7916  * Returns 0 on success or negative on failure
7917  **/
7918 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
7919 {
7920         int ret = -ENOENT;
7921         struct i40e_pf *pf = vsi->back;
7922
7923         if (vsi->q_vectors[0]) {
7924                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
7925                          vsi->seid);
7926                 return -EEXIST;
7927         }
7928
7929         if (vsi->base_vector) {
7930                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
7931                          vsi->seid, vsi->base_vector);
7932                 return -EEXIST;
7933         }
7934
7935         ret = i40e_vsi_alloc_q_vectors(vsi);
7936         if (ret) {
7937                 dev_info(&pf->pdev->dev,
7938                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
7939                          vsi->num_q_vectors, vsi->seid, ret);
7940                 vsi->num_q_vectors = 0;
7941                 goto vector_setup_out;
7942         }
7943
7944         if (vsi->num_q_vectors)
7945                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
7946                                                  vsi->num_q_vectors, vsi->idx);
7947         if (vsi->base_vector < 0) {
7948                 dev_info(&pf->pdev->dev,
7949                          "failed to get queue tracking for VSI %d, err=%d\n",
7950                          vsi->seid, vsi->base_vector);
7951                 i40e_vsi_free_q_vectors(vsi);
7952                 ret = -ENOENT;
7953                 goto vector_setup_out;
7954         }
7955
7956 vector_setup_out:
7957         return ret;
7958 }
7959
7960 /**
7961  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
7962  * @vsi: pointer to the vsi.
7963  *
7964  * This re-allocates a vsi's queue resources.
7965  *
7966  * Returns pointer to the successfully allocated and configured VSI sw struct
7967  * on success, otherwise returns NULL on failure.
7968  **/
7969 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
7970 {
7971         struct i40e_pf *pf = vsi->back;
7972         u8 enabled_tc;
7973         int ret;
7974
7975         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7976         i40e_vsi_clear_rings(vsi);
7977
7978         i40e_vsi_free_arrays(vsi, false);
7979         i40e_set_num_rings_in_vsi(vsi);
7980         ret = i40e_vsi_alloc_arrays(vsi, false);
7981         if (ret)
7982                 goto err_vsi;
7983
7984         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
7985         if (ret < 0) {
7986                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
7987                          vsi->seid, ret);
7988                 goto err_vsi;
7989         }
7990         vsi->base_queue = ret;
7991
7992         /* Update the FW view of the VSI. Force a reset of TC and queue
7993          * layout configurations.
7994          */
7995         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
7996         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
7997         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
7998         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
7999
8000         /* assign it some queues */
8001         ret = i40e_alloc_rings(vsi);
8002         if (ret)
8003                 goto err_rings;
8004
8005         /* map all of the rings to the q_vectors */
8006         i40e_vsi_map_rings_to_vectors(vsi);
8007         return vsi;
8008
8009 err_rings:
8010         i40e_vsi_free_q_vectors(vsi);
8011         if (vsi->netdev_registered) {
8012                 vsi->netdev_registered = false;
8013                 unregister_netdev(vsi->netdev);
8014                 free_netdev(vsi->netdev);
8015                 vsi->netdev = NULL;
8016         }
8017         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
8018 err_vsi:
8019         i40e_vsi_clear(vsi);
8020         return NULL;
8021 }
8022
8023 /**
8024  * i40e_vsi_setup - Set up a VSI by a given type
8025  * @pf: board private structure
8026  * @type: VSI type
8027  * @uplink_seid: the switch element to link to
8028  * @param1: usage depends upon VSI type. For VF types, indicates VF id
8029  *
8030  * This allocates the sw VSI structure and its queue resources, then add a VSI
8031  * to the identified VEB.
8032  *
8033  * Returns pointer to the successfully allocated and configure VSI sw struct on
8034  * success, otherwise returns NULL on failure.
8035  **/
8036 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
8037                                 u16 uplink_seid, u32 param1)
8038 {
8039         struct i40e_vsi *vsi = NULL;
8040         struct i40e_veb *veb = NULL;
8041         int ret, i;
8042         int v_idx;
8043
8044         /* The requested uplink_seid must be either
8045          *     - the PF's port seid
8046          *              no VEB is needed because this is the PF
8047          *              or this is a Flow Director special case VSI
8048          *     - seid of an existing VEB
8049          *     - seid of a VSI that owns an existing VEB
8050          *     - seid of a VSI that doesn't own a VEB
8051          *              a new VEB is created and the VSI becomes the owner
8052          *     - seid of the PF VSI, which is what creates the first VEB
8053          *              this is a special case of the previous
8054          *
8055          * Find which uplink_seid we were given and create a new VEB if needed
8056          */
8057         for (i = 0; i < I40E_MAX_VEB; i++) {
8058                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
8059                         veb = pf->veb[i];
8060                         break;
8061                 }
8062         }
8063
8064         if (!veb && uplink_seid != pf->mac_seid) {
8065
8066                 for (i = 0; i < pf->num_alloc_vsi; i++) {
8067                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
8068                                 vsi = pf->vsi[i];
8069                                 break;
8070                         }
8071                 }
8072                 if (!vsi) {
8073                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
8074                                  uplink_seid);
8075                         return NULL;
8076                 }
8077
8078                 if (vsi->uplink_seid == pf->mac_seid)
8079                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
8080                                              vsi->tc_config.enabled_tc);
8081                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
8082                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
8083                                              vsi->tc_config.enabled_tc);
8084
8085                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
8086                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
8087                                 veb = pf->veb[i];
8088                 }
8089                 if (!veb) {
8090                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
8091                         return NULL;
8092                 }
8093
8094                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
8095                 uplink_seid = veb->seid;
8096         }
8097
8098         /* get vsi sw struct */
8099         v_idx = i40e_vsi_mem_alloc(pf, type);
8100         if (v_idx < 0)
8101                 goto err_alloc;
8102         vsi = pf->vsi[v_idx];
8103         if (!vsi)
8104                 goto err_alloc;
8105         vsi->type = type;
8106         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
8107
8108         if (type == I40E_VSI_MAIN)
8109                 pf->lan_vsi = v_idx;
8110         else if (type == I40E_VSI_SRIOV)
8111                 vsi->vf_id = param1;
8112         /* assign it some queues */
8113         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
8114                                 vsi->idx);
8115         if (ret < 0) {
8116                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
8117                          vsi->seid, ret);
8118                 goto err_vsi;
8119         }
8120         vsi->base_queue = ret;
8121
8122         /* get a VSI from the hardware */
8123         vsi->uplink_seid = uplink_seid;
8124         ret = i40e_add_vsi(vsi);
8125         if (ret)
8126                 goto err_vsi;
8127
8128         switch (vsi->type) {
8129         /* setup the netdev if needed */
8130         case I40E_VSI_MAIN:
8131         case I40E_VSI_VMDQ2:
8132         case I40E_VSI_FCOE:
8133                 ret = i40e_config_netdev(vsi);
8134                 if (ret)
8135                         goto err_netdev;
8136                 ret = register_netdev(vsi->netdev);
8137                 if (ret)
8138                         goto err_netdev;
8139                 vsi->netdev_registered = true;
8140                 netif_carrier_off(vsi->netdev);
8141 #ifdef CONFIG_I40E_DCB
8142                 /* Setup DCB netlink interface */
8143                 i40e_dcbnl_setup(vsi);
8144 #endif /* CONFIG_I40E_DCB */
8145                 /* fall through */
8146
8147         case I40E_VSI_FDIR:
8148                 /* set up vectors and rings if needed */
8149                 ret = i40e_vsi_setup_vectors(vsi);
8150                 if (ret)
8151                         goto err_msix;
8152
8153                 ret = i40e_alloc_rings(vsi);
8154                 if (ret)
8155                         goto err_rings;
8156
8157                 /* map all of the rings to the q_vectors */
8158                 i40e_vsi_map_rings_to_vectors(vsi);
8159
8160                 i40e_vsi_reset_stats(vsi);
8161                 break;
8162
8163         default:
8164                 /* no netdev or rings for the other VSI types */
8165                 break;
8166         }
8167
8168         return vsi;
8169
8170 err_rings:
8171         i40e_vsi_free_q_vectors(vsi);
8172 err_msix:
8173         if (vsi->netdev_registered) {
8174                 vsi->netdev_registered = false;
8175                 unregister_netdev(vsi->netdev);
8176                 free_netdev(vsi->netdev);
8177                 vsi->netdev = NULL;
8178         }
8179 err_netdev:
8180         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
8181 err_vsi:
8182         i40e_vsi_clear(vsi);
8183 err_alloc:
8184         return NULL;
8185 }
8186
8187 /**
8188  * i40e_veb_get_bw_info - Query VEB BW information
8189  * @veb: the veb to query
8190  *
8191  * Query the Tx scheduler BW configuration data for given VEB
8192  **/
8193 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
8194 {
8195         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
8196         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
8197         struct i40e_pf *pf = veb->pf;
8198         struct i40e_hw *hw = &pf->hw;
8199         u32 tc_bw_max;
8200         int ret = 0;
8201         int i;
8202
8203         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
8204                                                   &bw_data, NULL);
8205         if (ret) {
8206                 dev_info(&pf->pdev->dev,
8207                          "query veb bw config failed, aq_err=%d\n",
8208                          hw->aq.asq_last_status);
8209                 goto out;
8210         }
8211
8212         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
8213                                                    &ets_data, NULL);
8214         if (ret) {
8215                 dev_info(&pf->pdev->dev,
8216                          "query veb bw ets config failed, aq_err=%d\n",
8217                          hw->aq.asq_last_status);
8218                 goto out;
8219         }
8220
8221         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
8222         veb->bw_max_quanta = ets_data.tc_bw_max;
8223         veb->is_abs_credits = bw_data.absolute_credits_enable;
8224         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
8225                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
8226         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8227                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
8228                 veb->bw_tc_limit_credits[i] =
8229                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
8230                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
8231         }
8232
8233 out:
8234         return ret;
8235 }
8236
8237 /**
8238  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
8239  * @pf: board private structure
8240  *
8241  * On error: returns error code (negative)
8242  * On success: returns vsi index in PF (positive)
8243  **/
8244 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
8245 {
8246         int ret = -ENOENT;
8247         struct i40e_veb *veb;
8248         int i;
8249
8250         /* Need to protect the allocation of switch elements at the PF level */
8251         mutex_lock(&pf->switch_mutex);
8252
8253         /* VEB list may be fragmented if VEB creation/destruction has
8254          * been happening.  We can afford to do a quick scan to look
8255          * for any free slots in the list.
8256          *
8257          * find next empty veb slot, looping back around if necessary
8258          */
8259         i = 0;
8260         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
8261                 i++;
8262         if (i >= I40E_MAX_VEB) {
8263                 ret = -ENOMEM;
8264                 goto err_alloc_veb;  /* out of VEB slots! */
8265         }
8266
8267         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
8268         if (!veb) {
8269                 ret = -ENOMEM;
8270                 goto err_alloc_veb;
8271         }
8272         veb->pf = pf;
8273         veb->idx = i;
8274         veb->enabled_tc = 1;
8275
8276         pf->veb[i] = veb;
8277         ret = i;
8278 err_alloc_veb:
8279         mutex_unlock(&pf->switch_mutex);
8280         return ret;
8281 }
8282
8283 /**
8284  * i40e_switch_branch_release - Delete a branch of the switch tree
8285  * @branch: where to start deleting
8286  *
8287  * This uses recursion to find the tips of the branch to be
8288  * removed, deleting until we get back to and can delete this VEB.
8289  **/
8290 static void i40e_switch_branch_release(struct i40e_veb *branch)
8291 {
8292         struct i40e_pf *pf = branch->pf;
8293         u16 branch_seid = branch->seid;
8294         u16 veb_idx = branch->idx;
8295         int i;
8296
8297         /* release any VEBs on this VEB - RECURSION */
8298         for (i = 0; i < I40E_MAX_VEB; i++) {
8299                 if (!pf->veb[i])
8300                         continue;
8301                 if (pf->veb[i]->uplink_seid == branch->seid)
8302                         i40e_switch_branch_release(pf->veb[i]);
8303         }
8304
8305         /* Release the VSIs on this VEB, but not the owner VSI.
8306          *
8307          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
8308          *       the VEB itself, so don't use (*branch) after this loop.
8309          */
8310         for (i = 0; i < pf->num_alloc_vsi; i++) {
8311                 if (!pf->vsi[i])
8312                         continue;
8313                 if (pf->vsi[i]->uplink_seid == branch_seid &&
8314                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
8315                         i40e_vsi_release(pf->vsi[i]);
8316                 }
8317         }
8318
8319         /* There's one corner case where the VEB might not have been
8320          * removed, so double check it here and remove it if needed.
8321          * This case happens if the veb was created from the debugfs
8322          * commands and no VSIs were added to it.
8323          */
8324         if (pf->veb[veb_idx])
8325                 i40e_veb_release(pf->veb[veb_idx]);
8326 }
8327
8328 /**
8329  * i40e_veb_clear - remove veb struct
8330  * @veb: the veb to remove
8331  **/
8332 static void i40e_veb_clear(struct i40e_veb *veb)
8333 {
8334         if (!veb)
8335                 return;
8336
8337         if (veb->pf) {
8338                 struct i40e_pf *pf = veb->pf;
8339
8340                 mutex_lock(&pf->switch_mutex);
8341                 if (pf->veb[veb->idx] == veb)
8342                         pf->veb[veb->idx] = NULL;
8343                 mutex_unlock(&pf->switch_mutex);
8344         }
8345
8346         kfree(veb);
8347 }
8348
8349 /**
8350  * i40e_veb_release - Delete a VEB and free its resources
8351  * @veb: the VEB being removed
8352  **/
8353 void i40e_veb_release(struct i40e_veb *veb)
8354 {
8355         struct i40e_vsi *vsi = NULL;
8356         struct i40e_pf *pf;
8357         int i, n = 0;
8358
8359         pf = veb->pf;
8360
8361         /* find the remaining VSI and check for extras */
8362         for (i = 0; i < pf->num_alloc_vsi; i++) {
8363                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
8364                         n++;
8365                         vsi = pf->vsi[i];
8366                 }
8367         }
8368         if (n != 1) {
8369                 dev_info(&pf->pdev->dev,
8370                          "can't remove VEB %d with %d VSIs left\n",
8371                          veb->seid, n);
8372                 return;
8373         }
8374
8375         /* move the remaining VSI to uplink veb */
8376         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
8377         if (veb->uplink_seid) {
8378                 vsi->uplink_seid = veb->uplink_seid;
8379                 if (veb->uplink_seid == pf->mac_seid)
8380                         vsi->veb_idx = I40E_NO_VEB;
8381                 else
8382                         vsi->veb_idx = veb->veb_idx;
8383         } else {
8384                 /* floating VEB */
8385                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
8386                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
8387         }
8388
8389         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
8390         i40e_veb_clear(veb);
8391 }
8392
8393 /**
8394  * i40e_add_veb - create the VEB in the switch
8395  * @veb: the VEB to be instantiated
8396  * @vsi: the controlling VSI
8397  **/
8398 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
8399 {
8400         bool is_default = false;
8401         bool is_cloud = false;
8402         int ret;
8403
8404         /* get a VEB from the hardware */
8405         ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
8406                               veb->enabled_tc, is_default,
8407                               is_cloud, &veb->seid, NULL);
8408         if (ret) {
8409                 dev_info(&veb->pf->pdev->dev,
8410                          "couldn't add VEB, err %d, aq_err %d\n",
8411                          ret, veb->pf->hw.aq.asq_last_status);
8412                 return -EPERM;
8413         }
8414
8415         /* get statistics counter */
8416         ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
8417                                          &veb->stats_idx, NULL, NULL, NULL);
8418         if (ret) {
8419                 dev_info(&veb->pf->pdev->dev,
8420                          "couldn't get VEB statistics idx, err %d, aq_err %d\n",
8421                          ret, veb->pf->hw.aq.asq_last_status);
8422                 return -EPERM;
8423         }
8424         ret = i40e_veb_get_bw_info(veb);
8425         if (ret) {
8426                 dev_info(&veb->pf->pdev->dev,
8427                          "couldn't get VEB bw info, err %d, aq_err %d\n",
8428                          ret, veb->pf->hw.aq.asq_last_status);
8429                 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
8430                 return -ENOENT;
8431         }
8432
8433         vsi->uplink_seid = veb->seid;
8434         vsi->veb_idx = veb->idx;
8435         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
8436
8437         return 0;
8438 }
8439
8440 /**
8441  * i40e_veb_setup - Set up a VEB
8442  * @pf: board private structure
8443  * @flags: VEB setup flags
8444  * @uplink_seid: the switch element to link to
8445  * @vsi_seid: the initial VSI seid
8446  * @enabled_tc: Enabled TC bit-map
8447  *
8448  * This allocates the sw VEB structure and links it into the switch
8449  * It is possible and legal for this to be a duplicate of an already
8450  * existing VEB.  It is also possible for both uplink and vsi seids
8451  * to be zero, in order to create a floating VEB.
8452  *
8453  * Returns pointer to the successfully allocated VEB sw struct on
8454  * success, otherwise returns NULL on failure.
8455  **/
8456 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
8457                                 u16 uplink_seid, u16 vsi_seid,
8458                                 u8 enabled_tc)
8459 {
8460         struct i40e_veb *veb, *uplink_veb = NULL;
8461         int vsi_idx, veb_idx;
8462         int ret;
8463
8464         /* if one seid is 0, the other must be 0 to create a floating relay */
8465         if ((uplink_seid == 0 || vsi_seid == 0) &&
8466             (uplink_seid + vsi_seid != 0)) {
8467                 dev_info(&pf->pdev->dev,
8468                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
8469                          uplink_seid, vsi_seid);
8470                 return NULL;
8471         }
8472
8473         /* make sure there is such a vsi and uplink */
8474         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
8475                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
8476                         break;
8477         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
8478                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
8479                          vsi_seid);
8480                 return NULL;
8481         }
8482
8483         if (uplink_seid && uplink_seid != pf->mac_seid) {
8484                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
8485                         if (pf->veb[veb_idx] &&
8486                             pf->veb[veb_idx]->seid == uplink_seid) {
8487                                 uplink_veb = pf->veb[veb_idx];
8488                                 break;
8489                         }
8490                 }
8491                 if (!uplink_veb) {
8492                         dev_info(&pf->pdev->dev,
8493                                  "uplink seid %d not found\n", uplink_seid);
8494                         return NULL;
8495                 }
8496         }
8497
8498         /* get veb sw struct */
8499         veb_idx = i40e_veb_mem_alloc(pf);
8500         if (veb_idx < 0)
8501                 goto err_alloc;
8502         veb = pf->veb[veb_idx];
8503         veb->flags = flags;
8504         veb->uplink_seid = uplink_seid;
8505         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
8506         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
8507
8508         /* create the VEB in the switch */
8509         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
8510         if (ret)
8511                 goto err_veb;
8512         if (vsi_idx == pf->lan_vsi)
8513                 pf->lan_veb = veb->idx;
8514
8515         return veb;
8516
8517 err_veb:
8518         i40e_veb_clear(veb);
8519 err_alloc:
8520         return NULL;
8521 }
8522
8523 /**
8524  * i40e_setup_pf_switch_element - set pf vars based on switch type
8525  * @pf: board private structure
8526  * @ele: element we are building info from
8527  * @num_reported: total number of elements
8528  * @printconfig: should we print the contents
8529  *
8530  * helper function to assist in extracting a few useful SEID values.
8531  **/
8532 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
8533                                 struct i40e_aqc_switch_config_element_resp *ele,
8534                                 u16 num_reported, bool printconfig)
8535 {
8536         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
8537         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
8538         u8 element_type = ele->element_type;
8539         u16 seid = le16_to_cpu(ele->seid);
8540
8541         if (printconfig)
8542                 dev_info(&pf->pdev->dev,
8543                          "type=%d seid=%d uplink=%d downlink=%d\n",
8544                          element_type, seid, uplink_seid, downlink_seid);
8545
8546         switch (element_type) {
8547         case I40E_SWITCH_ELEMENT_TYPE_MAC:
8548                 pf->mac_seid = seid;
8549                 break;
8550         case I40E_SWITCH_ELEMENT_TYPE_VEB:
8551                 /* Main VEB? */
8552                 if (uplink_seid != pf->mac_seid)
8553                         break;
8554                 if (pf->lan_veb == I40E_NO_VEB) {
8555                         int v;
8556
8557                         /* find existing or else empty VEB */
8558                         for (v = 0; v < I40E_MAX_VEB; v++) {
8559                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
8560                                         pf->lan_veb = v;
8561                                         break;
8562                                 }
8563                         }
8564                         if (pf->lan_veb == I40E_NO_VEB) {
8565                                 v = i40e_veb_mem_alloc(pf);
8566                                 if (v < 0)
8567                                         break;
8568                                 pf->lan_veb = v;
8569                         }
8570                 }
8571
8572                 pf->veb[pf->lan_veb]->seid = seid;
8573                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
8574                 pf->veb[pf->lan_veb]->pf = pf;
8575                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
8576                 break;
8577         case I40E_SWITCH_ELEMENT_TYPE_VSI:
8578                 if (num_reported != 1)
8579                         break;
8580                 /* This is immediately after a reset so we can assume this is
8581                  * the PF's VSI
8582                  */
8583                 pf->mac_seid = uplink_seid;
8584                 pf->pf_seid = downlink_seid;
8585                 pf->main_vsi_seid = seid;
8586                 if (printconfig)
8587                         dev_info(&pf->pdev->dev,
8588                                  "pf_seid=%d main_vsi_seid=%d\n",
8589                                  pf->pf_seid, pf->main_vsi_seid);
8590                 break;
8591         case I40E_SWITCH_ELEMENT_TYPE_PF:
8592         case I40E_SWITCH_ELEMENT_TYPE_VF:
8593         case I40E_SWITCH_ELEMENT_TYPE_EMP:
8594         case I40E_SWITCH_ELEMENT_TYPE_BMC:
8595         case I40E_SWITCH_ELEMENT_TYPE_PE:
8596         case I40E_SWITCH_ELEMENT_TYPE_PA:
8597                 /* ignore these for now */
8598                 break;
8599         default:
8600                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
8601                          element_type, seid);
8602                 break;
8603         }
8604 }
8605
8606 /**
8607  * i40e_fetch_switch_configuration - Get switch config from firmware
8608  * @pf: board private structure
8609  * @printconfig: should we print the contents
8610  *
8611  * Get the current switch configuration from the device and
8612  * extract a few useful SEID values.
8613  **/
8614 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
8615 {
8616         struct i40e_aqc_get_switch_config_resp *sw_config;
8617         u16 next_seid = 0;
8618         int ret = 0;
8619         u8 *aq_buf;
8620         int i;
8621
8622         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
8623         if (!aq_buf)
8624                 return -ENOMEM;
8625
8626         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
8627         do {
8628                 u16 num_reported, num_total;
8629
8630                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
8631                                                 I40E_AQ_LARGE_BUF,
8632                                                 &next_seid, NULL);
8633                 if (ret) {
8634                         dev_info(&pf->pdev->dev,
8635                                  "get switch config failed %d aq_err=%x\n",
8636                                  ret, pf->hw.aq.asq_last_status);
8637                         kfree(aq_buf);
8638                         return -ENOENT;
8639                 }
8640
8641                 num_reported = le16_to_cpu(sw_config->header.num_reported);
8642                 num_total = le16_to_cpu(sw_config->header.num_total);
8643
8644                 if (printconfig)
8645                         dev_info(&pf->pdev->dev,
8646                                  "header: %d reported %d total\n",
8647                                  num_reported, num_total);
8648
8649                 for (i = 0; i < num_reported; i++) {
8650                         struct i40e_aqc_switch_config_element_resp *ele =
8651                                 &sw_config->element[i];
8652
8653                         i40e_setup_pf_switch_element(pf, ele, num_reported,
8654                                                      printconfig);
8655                 }
8656         } while (next_seid != 0);
8657
8658         kfree(aq_buf);
8659         return ret;
8660 }
8661
8662 /**
8663  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
8664  * @pf: board private structure
8665  * @reinit: if the Main VSI needs to re-initialized.
8666  *
8667  * Returns 0 on success, negative value on failure
8668  **/
8669 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
8670 {
8671         int ret;
8672
8673         /* find out what's out there already */
8674         ret = i40e_fetch_switch_configuration(pf, false);
8675         if (ret) {
8676                 dev_info(&pf->pdev->dev,
8677                          "couldn't fetch switch config, err %d, aq_err %d\n",
8678                          ret, pf->hw.aq.asq_last_status);
8679                 return ret;
8680         }
8681         i40e_pf_reset_stats(pf);
8682
8683         /* first time setup */
8684         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
8685                 struct i40e_vsi *vsi = NULL;
8686                 u16 uplink_seid;
8687
8688                 /* Set up the PF VSI associated with the PF's main VSI
8689                  * that is already in the HW switch
8690                  */
8691                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
8692                         uplink_seid = pf->veb[pf->lan_veb]->seid;
8693                 else
8694                         uplink_seid = pf->mac_seid;
8695                 if (pf->lan_vsi == I40E_NO_VSI)
8696                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
8697                 else if (reinit)
8698                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
8699                 if (!vsi) {
8700                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
8701                         i40e_fdir_teardown(pf);
8702                         return -EAGAIN;
8703                 }
8704         } else {
8705                 /* force a reset of TC and queue layout configurations */
8706                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
8707                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
8708                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
8709                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
8710         }
8711         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
8712
8713         i40e_fdir_sb_setup(pf);
8714
8715         /* Setup static PF queue filter control settings */
8716         ret = i40e_setup_pf_filter_control(pf);
8717         if (ret) {
8718                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
8719                          ret);
8720                 /* Failure here should not stop continuing other steps */
8721         }
8722
8723         /* enable RSS in the HW, even for only one queue, as the stack can use
8724          * the hash
8725          */
8726         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
8727                 i40e_config_rss(pf);
8728
8729         /* fill in link information and enable LSE reporting */
8730         i40e_update_link_info(&pf->hw, true);
8731         i40e_link_event(pf);
8732
8733         /* Initialize user-specific link properties */
8734         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
8735                                   I40E_AQ_AN_COMPLETED) ? true : false);
8736
8737         /* fill in link information and enable LSE reporting */
8738         i40e_update_link_info(&pf->hw, true);
8739         i40e_link_event(pf);
8740
8741         /* Initialize user-specific link properties */
8742         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
8743                                   I40E_AQ_AN_COMPLETED) ? true : false);
8744
8745         i40e_ptp_init(pf);
8746
8747         return ret;
8748 }
8749
8750 /**
8751  * i40e_determine_queue_usage - Work out queue distribution
8752  * @pf: board private structure
8753  **/
8754 static void i40e_determine_queue_usage(struct i40e_pf *pf)
8755 {
8756         int queues_left;
8757
8758         pf->num_lan_qps = 0;
8759 #ifdef I40E_FCOE
8760         pf->num_fcoe_qps = 0;
8761 #endif
8762
8763         /* Find the max queues to be put into basic use.  We'll always be
8764          * using TC0, whether or not DCB is running, and TC0 will get the
8765          * big RSS set.
8766          */
8767         queues_left = pf->hw.func_caps.num_tx_qp;
8768
8769         if ((queues_left == 1) ||
8770             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
8771                 /* one qp for PF, no queues for anything else */
8772                 queues_left = 0;
8773                 pf->rss_size = pf->num_lan_qps = 1;
8774
8775                 /* make sure all the fancies are disabled */
8776                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
8777 #ifdef I40E_FCOE
8778                                I40E_FLAG_FCOE_ENABLED   |
8779 #endif
8780                                I40E_FLAG_FD_SB_ENABLED  |
8781                                I40E_FLAG_FD_ATR_ENABLED |
8782                                I40E_FLAG_DCB_CAPABLE    |
8783                                I40E_FLAG_SRIOV_ENABLED  |
8784                                I40E_FLAG_VMDQ_ENABLED);
8785         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
8786                                   I40E_FLAG_FD_SB_ENABLED |
8787                                   I40E_FLAG_FD_ATR_ENABLED |
8788                                   I40E_FLAG_DCB_CAPABLE))) {
8789                 /* one qp for PF */
8790                 pf->rss_size = pf->num_lan_qps = 1;
8791                 queues_left -= pf->num_lan_qps;
8792
8793                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
8794 #ifdef I40E_FCOE
8795                                I40E_FLAG_FCOE_ENABLED   |
8796 #endif
8797                                I40E_FLAG_FD_SB_ENABLED  |
8798                                I40E_FLAG_FD_ATR_ENABLED |
8799                                I40E_FLAG_DCB_ENABLED    |
8800                                I40E_FLAG_VMDQ_ENABLED);
8801         } else {
8802                 /* Not enough queues for all TCs */
8803                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
8804                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
8805                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
8806                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
8807                 }
8808                 pf->num_lan_qps = pf->rss_size_max;
8809                 queues_left -= pf->num_lan_qps;
8810         }
8811
8812 #ifdef I40E_FCOE
8813         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
8814                 if (I40E_DEFAULT_FCOE <= queues_left) {
8815                         pf->num_fcoe_qps = I40E_DEFAULT_FCOE;
8816                 } else if (I40E_MINIMUM_FCOE <= queues_left) {
8817                         pf->num_fcoe_qps = I40E_MINIMUM_FCOE;
8818                 } else {
8819                         pf->num_fcoe_qps = 0;
8820                         pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
8821                         dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n");
8822                 }
8823
8824                 queues_left -= pf->num_fcoe_qps;
8825         }
8826
8827 #endif
8828         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8829                 if (queues_left > 1) {
8830                         queues_left -= 1; /* save 1 queue for FD */
8831                 } else {
8832                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8833                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
8834                 }
8835         }
8836
8837         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
8838             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
8839                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
8840                                         (queues_left / pf->num_vf_qps));
8841                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
8842         }
8843
8844         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8845             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
8846                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
8847                                           (queues_left / pf->num_vmdq_qps));
8848                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
8849         }
8850
8851         pf->queues_left = queues_left;
8852 #ifdef I40E_FCOE
8853         dev_info(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps);
8854 #endif
8855 }
8856
8857 /**
8858  * i40e_setup_pf_filter_control - Setup PF static filter control
8859  * @pf: PF to be setup
8860  *
8861  * i40e_setup_pf_filter_control sets up a pf's initial filter control
8862  * settings. If PE/FCoE are enabled then it will also set the per PF
8863  * based filter sizes required for them. It also enables Flow director,
8864  * ethertype and macvlan type filter settings for the pf.
8865  *
8866  * Returns 0 on success, negative on failure
8867  **/
8868 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
8869 {
8870         struct i40e_filter_control_settings *settings = &pf->filter_settings;
8871
8872         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
8873
8874         /* Flow Director is enabled */
8875         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
8876                 settings->enable_fdir = true;
8877
8878         /* Ethtype and MACVLAN filters enabled for PF */
8879         settings->enable_ethtype = true;
8880         settings->enable_macvlan = true;
8881
8882         if (i40e_set_filter_control(&pf->hw, settings))
8883                 return -ENOENT;
8884
8885         return 0;
8886 }
8887
8888 #define INFO_STRING_LEN 255
8889 static void i40e_print_features(struct i40e_pf *pf)
8890 {
8891         struct i40e_hw *hw = &pf->hw;
8892         char *buf, *string;
8893
8894         string = kzalloc(INFO_STRING_LEN, GFP_KERNEL);
8895         if (!string) {
8896                 dev_err(&pf->pdev->dev, "Features string allocation failed\n");
8897                 return;
8898         }
8899
8900         buf = string;
8901
8902         buf += sprintf(string, "Features: PF-id[%d] ", hw->pf_id);
8903 #ifdef CONFIG_PCI_IOV
8904         buf += sprintf(buf, "VFs: %d ", pf->num_req_vfs);
8905 #endif
8906         buf += sprintf(buf, "VSIs: %d QP: %d ", pf->hw.func_caps.num_vsis,
8907                        pf->vsi[pf->lan_vsi]->num_queue_pairs);
8908
8909         if (pf->flags & I40E_FLAG_RSS_ENABLED)
8910                 buf += sprintf(buf, "RSS ");
8911         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
8912                 buf += sprintf(buf, "FD_ATR ");
8913         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8914                 buf += sprintf(buf, "FD_SB ");
8915                 buf += sprintf(buf, "NTUPLE ");
8916         }
8917         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
8918                 buf += sprintf(buf, "DCB ");
8919         if (pf->flags & I40E_FLAG_PTP)
8920                 buf += sprintf(buf, "PTP ");
8921 #ifdef I40E_FCOE
8922         if (pf->flags & I40E_FLAG_FCOE_ENABLED)
8923                 buf += sprintf(buf, "FCOE ");
8924 #endif
8925
8926         BUG_ON(buf > (string + INFO_STRING_LEN));
8927         dev_info(&pf->pdev->dev, "%s\n", string);
8928         kfree(string);
8929 }
8930
8931 /**
8932  * i40e_probe - Device initialization routine
8933  * @pdev: PCI device information struct
8934  * @ent: entry in i40e_pci_tbl
8935  *
8936  * i40e_probe initializes a pf identified by a pci_dev structure.
8937  * The OS initialization, configuring of the pf private structure,
8938  * and a hardware reset occur.
8939  *
8940  * Returns 0 on success, negative on failure
8941  **/
8942 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
8943 {
8944         struct i40e_pf *pf;
8945         struct i40e_hw *hw;
8946         static u16 pfs_found;
8947         u16 link_status;
8948         int err = 0;
8949         u32 len;
8950         u32 i;
8951
8952         err = pci_enable_device_mem(pdev);
8953         if (err)
8954                 return err;
8955
8956         /* set up for high or low dma */
8957         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
8958         if (err) {
8959                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
8960                 if (err) {
8961                         dev_err(&pdev->dev,
8962                                 "DMA configuration failed: 0x%x\n", err);
8963                         goto err_dma;
8964                 }
8965         }
8966
8967         /* set up pci connections */
8968         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
8969                                            IORESOURCE_MEM), i40e_driver_name);
8970         if (err) {
8971                 dev_info(&pdev->dev,
8972                          "pci_request_selected_regions failed %d\n", err);
8973                 goto err_pci_reg;
8974         }
8975
8976         pci_enable_pcie_error_reporting(pdev);
8977         pci_set_master(pdev);
8978
8979         /* Now that we have a PCI connection, we need to do the
8980          * low level device setup.  This is primarily setting up
8981          * the Admin Queue structures and then querying for the
8982          * device's current profile information.
8983          */
8984         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
8985         if (!pf) {
8986                 err = -ENOMEM;
8987                 goto err_pf_alloc;
8988         }
8989         pf->next_vsi = 0;
8990         pf->pdev = pdev;
8991         set_bit(__I40E_DOWN, &pf->state);
8992
8993         hw = &pf->hw;
8994         hw->back = pf;
8995         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8996                               pci_resource_len(pdev, 0));
8997         if (!hw->hw_addr) {
8998                 err = -EIO;
8999                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
9000                          (unsigned int)pci_resource_start(pdev, 0),
9001                          (unsigned int)pci_resource_len(pdev, 0), err);
9002                 goto err_ioremap;
9003         }
9004         hw->vendor_id = pdev->vendor;
9005         hw->device_id = pdev->device;
9006         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
9007         hw->subsystem_vendor_id = pdev->subsystem_vendor;
9008         hw->subsystem_device_id = pdev->subsystem_device;
9009         hw->bus.device = PCI_SLOT(pdev->devfn);
9010         hw->bus.func = PCI_FUNC(pdev->devfn);
9011         pf->instance = pfs_found;
9012
9013         /* do a special CORER for clearing PXE mode once at init */
9014         if (hw->revision_id == 0 &&
9015             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
9016                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
9017                 i40e_flush(hw);
9018                 msleep(200);
9019                 pf->corer_count++;
9020
9021                 i40e_clear_pxe_mode(hw);
9022         }
9023
9024         /* Reset here to make sure all is clean and to define PF 'n' */
9025         i40e_clear_hw(hw);
9026         err = i40e_pf_reset(hw);
9027         if (err) {
9028                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
9029                 goto err_pf_reset;
9030         }
9031         pf->pfr_count++;
9032
9033         hw->aq.num_arq_entries = I40E_AQ_LEN;
9034         hw->aq.num_asq_entries = I40E_AQ_LEN;
9035         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
9036         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
9037         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
9038         snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
9039                  "%s-pf%d:misc",
9040                  dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
9041
9042         err = i40e_init_shared_code(hw);
9043         if (err) {
9044                 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
9045                 goto err_pf_reset;
9046         }
9047
9048         /* set up a default setting for link flow control */
9049         pf->hw.fc.requested_mode = I40E_FC_NONE;
9050
9051         err = i40e_init_adminq(hw);
9052         dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
9053         if (err) {
9054                 dev_info(&pdev->dev,
9055                          "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
9056                 goto err_pf_reset;
9057         }
9058
9059         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
9060             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
9061                 dev_info(&pdev->dev,
9062                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
9063         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
9064                  hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
9065                 dev_info(&pdev->dev,
9066                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
9067
9068
9069         i40e_verify_eeprom(pf);
9070
9071         /* Rev 0 hardware was never productized */
9072         if (hw->revision_id < 1)
9073                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
9074
9075         i40e_clear_pxe_mode(hw);
9076         err = i40e_get_capabilities(pf);
9077         if (err)
9078                 goto err_adminq_setup;
9079
9080         err = i40e_sw_init(pf);
9081         if (err) {
9082                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
9083                 goto err_sw_init;
9084         }
9085
9086         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
9087                                 hw->func_caps.num_rx_qp,
9088                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
9089         if (err) {
9090                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
9091                 goto err_init_lan_hmc;
9092         }
9093
9094         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
9095         if (err) {
9096                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
9097                 err = -ENOENT;
9098                 goto err_configure_lan_hmc;
9099         }
9100
9101         i40e_get_mac_addr(hw, hw->mac.addr);
9102         if (!is_valid_ether_addr(hw->mac.addr)) {
9103                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
9104                 err = -EIO;
9105                 goto err_mac_addr;
9106         }
9107         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
9108         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
9109         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
9110         if (is_valid_ether_addr(hw->mac.port_addr))
9111                 pf->flags |= I40E_FLAG_PORT_ID_VALID;
9112 #ifdef I40E_FCOE
9113         err = i40e_get_san_mac_addr(hw, hw->mac.san_addr);
9114         if (err)
9115                 dev_info(&pdev->dev,
9116                          "(non-fatal) SAN MAC retrieval failed: %d\n", err);
9117         if (!is_valid_ether_addr(hw->mac.san_addr)) {
9118                 dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n",
9119                          hw->mac.san_addr);
9120                 ether_addr_copy(hw->mac.san_addr, hw->mac.addr);
9121         }
9122         dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr);
9123 #endif /* I40E_FCOE */
9124
9125         pci_set_drvdata(pdev, pf);
9126         pci_save_state(pdev);
9127 #ifdef CONFIG_I40E_DCB
9128         err = i40e_init_pf_dcb(pf);
9129         if (err) {
9130                 dev_info(&pdev->dev, "init_pf_dcb failed: %d\n", err);
9131                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9132                 /* Continue without DCB enabled */
9133         }
9134 #endif /* CONFIG_I40E_DCB */
9135
9136         /* set up periodic task facility */
9137         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
9138         pf->service_timer_period = HZ;
9139
9140         INIT_WORK(&pf->service_task, i40e_service_task);
9141         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
9142         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
9143         pf->link_check_timeout = jiffies;
9144
9145         /* WoL defaults to disabled */
9146         pf->wol_en = false;
9147         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
9148
9149         /* set up the main switch operations */
9150         i40e_determine_queue_usage(pf);
9151         i40e_init_interrupt_scheme(pf);
9152
9153         /* The number of VSIs reported by the FW is the minimum guaranteed
9154          * to us; HW supports far more and we share the remaining pool with
9155          * the other PFs. We allocate space for more than the guarantee with
9156          * the understanding that we might not get them all later.
9157          */
9158         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
9159                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
9160         else
9161                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
9162
9163         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
9164         len = sizeof(struct i40e_vsi *) * pf->num_alloc_vsi;
9165         pf->vsi = kzalloc(len, GFP_KERNEL);
9166         if (!pf->vsi) {
9167                 err = -ENOMEM;
9168                 goto err_switch_setup;
9169         }
9170
9171         err = i40e_setup_pf_switch(pf, false);
9172         if (err) {
9173                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
9174                 goto err_vsis;
9175         }
9176         /* if FDIR VSI was set up, start it now */
9177         for (i = 0; i < pf->num_alloc_vsi; i++) {
9178                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
9179                         i40e_vsi_open(pf->vsi[i]);
9180                         break;
9181                 }
9182         }
9183
9184         /* driver is only interested in link up/down and module qualification
9185          * reports from firmware
9186          */
9187         err = i40e_aq_set_phy_int_mask(&pf->hw,
9188                                        I40E_AQ_EVENT_LINK_UPDOWN |
9189                                        I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
9190         if (err)
9191                 dev_info(&pf->pdev->dev, "set phy mask fail, aq_err %d\n", err);
9192
9193         msleep(75);
9194         err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
9195         if (err) {
9196                 dev_info(&pf->pdev->dev, "link restart failed, aq_err=%d\n",
9197                          pf->hw.aq.asq_last_status);
9198         }
9199
9200         /* The main driver is (mostly) up and happy. We need to set this state
9201          * before setting up the misc vector or we get a race and the vector
9202          * ends up disabled forever.
9203          */
9204         clear_bit(__I40E_DOWN, &pf->state);
9205
9206         /* In case of MSIX we are going to setup the misc vector right here
9207          * to handle admin queue events etc. In case of legacy and MSI
9208          * the misc functionality and queue processing is combined in
9209          * the same vector and that gets setup at open.
9210          */
9211         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
9212                 err = i40e_setup_misc_vector(pf);
9213                 if (err) {
9214                         dev_info(&pdev->dev,
9215                                  "setup of misc vector failed: %d\n", err);
9216                         goto err_vsis;
9217                 }
9218         }
9219
9220 #ifdef CONFIG_PCI_IOV
9221         /* prep for VF support */
9222         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
9223             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
9224             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
9225                 u32 val;
9226
9227                 /* disable link interrupts for VFs */
9228                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
9229                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
9230                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
9231                 i40e_flush(hw);
9232
9233                 if (pci_num_vf(pdev)) {
9234                         dev_info(&pdev->dev,
9235                                  "Active VFs found, allocating resources.\n");
9236                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
9237                         if (err)
9238                                 dev_info(&pdev->dev,
9239                                          "Error %d allocating resources for existing VFs\n",
9240                                          err);
9241                 }
9242         }
9243 #endif /* CONFIG_PCI_IOV */
9244
9245         pfs_found++;
9246
9247         i40e_dbg_pf_init(pf);
9248
9249         /* tell the firmware that we're starting */
9250         i40e_send_version(pf);
9251
9252         /* since everything's happy, start the service_task timer */
9253         mod_timer(&pf->service_timer,
9254                   round_jiffies(jiffies + pf->service_timer_period));
9255
9256 #ifdef I40E_FCOE
9257         /* create FCoE interface */
9258         i40e_fcoe_vsi_setup(pf);
9259
9260 #endif
9261         /* Get the negotiated link width and speed from PCI config space */
9262         pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
9263
9264         i40e_set_pci_config_data(hw, link_status);
9265
9266         dev_info(&pdev->dev, "PCI-Express: %s %s\n",
9267                 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
9268                  hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
9269                  hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
9270                  "Unknown"),
9271                 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
9272                  hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
9273                  hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
9274                  hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
9275                  "Unknown"));
9276
9277         if (hw->bus.width < i40e_bus_width_pcie_x8 ||
9278             hw->bus.speed < i40e_bus_speed_8000) {
9279                 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
9280                 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
9281         }
9282
9283         /* print a string summarizing features */
9284         i40e_print_features(pf);
9285
9286         return 0;
9287
9288         /* Unwind what we've done if something failed in the setup */
9289 err_vsis:
9290         set_bit(__I40E_DOWN, &pf->state);
9291         i40e_clear_interrupt_scheme(pf);
9292         kfree(pf->vsi);
9293 err_switch_setup:
9294         i40e_reset_interrupt_capability(pf);
9295         del_timer_sync(&pf->service_timer);
9296 err_mac_addr:
9297 err_configure_lan_hmc:
9298         (void)i40e_shutdown_lan_hmc(hw);
9299 err_init_lan_hmc:
9300         kfree(pf->qp_pile);
9301         kfree(pf->irq_pile);
9302 err_sw_init:
9303 err_adminq_setup:
9304         (void)i40e_shutdown_adminq(hw);
9305 err_pf_reset:
9306         iounmap(hw->hw_addr);
9307 err_ioremap:
9308         kfree(pf);
9309 err_pf_alloc:
9310         pci_disable_pcie_error_reporting(pdev);
9311         pci_release_selected_regions(pdev,
9312                                      pci_select_bars(pdev, IORESOURCE_MEM));
9313 err_pci_reg:
9314 err_dma:
9315         pci_disable_device(pdev);
9316         return err;
9317 }
9318
9319 /**
9320  * i40e_remove - Device removal routine
9321  * @pdev: PCI device information struct
9322  *
9323  * i40e_remove is called by the PCI subsystem to alert the driver
9324  * that is should release a PCI device.  This could be caused by a
9325  * Hot-Plug event, or because the driver is going to be removed from
9326  * memory.
9327  **/
9328 static void i40e_remove(struct pci_dev *pdev)
9329 {
9330         struct i40e_pf *pf = pci_get_drvdata(pdev);
9331         i40e_status ret_code;
9332         int i;
9333
9334         i40e_dbg_pf_exit(pf);
9335
9336         i40e_ptp_stop(pf);
9337
9338         /* no more scheduling of any task */
9339         set_bit(__I40E_DOWN, &pf->state);
9340         del_timer_sync(&pf->service_timer);
9341         cancel_work_sync(&pf->service_task);
9342
9343         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
9344                 i40e_free_vfs(pf);
9345                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
9346         }
9347
9348         i40e_fdir_teardown(pf);
9349
9350         /* If there is a switch structure or any orphans, remove them.
9351          * This will leave only the PF's VSI remaining.
9352          */
9353         for (i = 0; i < I40E_MAX_VEB; i++) {
9354                 if (!pf->veb[i])
9355                         continue;
9356
9357                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
9358                     pf->veb[i]->uplink_seid == 0)
9359                         i40e_switch_branch_release(pf->veb[i]);
9360         }
9361
9362         /* Now we can shutdown the PF's VSI, just before we kill
9363          * adminq and hmc.
9364          */
9365         if (pf->vsi[pf->lan_vsi])
9366                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
9367
9368         i40e_stop_misc_vector(pf);
9369         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
9370                 synchronize_irq(pf->msix_entries[0].vector);
9371                 free_irq(pf->msix_entries[0].vector, pf);
9372         }
9373
9374         /* shutdown and destroy the HMC */
9375         if (pf->hw.hmc.hmc_obj) {
9376                 ret_code = i40e_shutdown_lan_hmc(&pf->hw);
9377                 if (ret_code)
9378                         dev_warn(&pdev->dev,
9379                                  "Failed to destroy the HMC resources: %d\n",
9380                                  ret_code);
9381         }
9382
9383         /* shutdown the adminq */
9384         ret_code = i40e_shutdown_adminq(&pf->hw);
9385         if (ret_code)
9386                 dev_warn(&pdev->dev,
9387                          "Failed to destroy the Admin Queue resources: %d\n",
9388                          ret_code);
9389
9390         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
9391         i40e_clear_interrupt_scheme(pf);
9392         for (i = 0; i < pf->num_alloc_vsi; i++) {
9393                 if (pf->vsi[i]) {
9394                         i40e_vsi_clear_rings(pf->vsi[i]);
9395                         i40e_vsi_clear(pf->vsi[i]);
9396                         pf->vsi[i] = NULL;
9397                 }
9398         }
9399
9400         for (i = 0; i < I40E_MAX_VEB; i++) {
9401                 kfree(pf->veb[i]);
9402                 pf->veb[i] = NULL;
9403         }
9404
9405         kfree(pf->qp_pile);
9406         kfree(pf->irq_pile);
9407         kfree(pf->vsi);
9408
9409         iounmap(pf->hw.hw_addr);
9410         kfree(pf);
9411         pci_release_selected_regions(pdev,
9412                                      pci_select_bars(pdev, IORESOURCE_MEM));
9413
9414         pci_disable_pcie_error_reporting(pdev);
9415         pci_disable_device(pdev);
9416 }
9417
9418 /**
9419  * i40e_pci_error_detected - warning that something funky happened in PCI land
9420  * @pdev: PCI device information struct
9421  *
9422  * Called to warn that something happened and the error handling steps
9423  * are in progress.  Allows the driver to quiesce things, be ready for
9424  * remediation.
9425  **/
9426 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
9427                                                 enum pci_channel_state error)
9428 {
9429         struct i40e_pf *pf = pci_get_drvdata(pdev);
9430
9431         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
9432
9433         /* shutdown all operations */
9434         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
9435                 rtnl_lock();
9436                 i40e_prep_for_reset(pf);
9437                 rtnl_unlock();
9438         }
9439
9440         /* Request a slot reset */
9441         return PCI_ERS_RESULT_NEED_RESET;
9442 }
9443
9444 /**
9445  * i40e_pci_error_slot_reset - a PCI slot reset just happened
9446  * @pdev: PCI device information struct
9447  *
9448  * Called to find if the driver can work with the device now that
9449  * the pci slot has been reset.  If a basic connection seems good
9450  * (registers are readable and have sane content) then return a
9451  * happy little PCI_ERS_RESULT_xxx.
9452  **/
9453 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
9454 {
9455         struct i40e_pf *pf = pci_get_drvdata(pdev);
9456         pci_ers_result_t result;
9457         int err;
9458         u32 reg;
9459
9460         dev_info(&pdev->dev, "%s\n", __func__);
9461         if (pci_enable_device_mem(pdev)) {
9462                 dev_info(&pdev->dev,
9463                          "Cannot re-enable PCI device after reset.\n");
9464                 result = PCI_ERS_RESULT_DISCONNECT;
9465         } else {
9466                 pci_set_master(pdev);
9467                 pci_restore_state(pdev);
9468                 pci_save_state(pdev);
9469                 pci_wake_from_d3(pdev, false);
9470
9471                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9472                 if (reg == 0)
9473                         result = PCI_ERS_RESULT_RECOVERED;
9474                 else
9475                         result = PCI_ERS_RESULT_DISCONNECT;
9476         }
9477
9478         err = pci_cleanup_aer_uncorrect_error_status(pdev);
9479         if (err) {
9480                 dev_info(&pdev->dev,
9481                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
9482                          err);
9483                 /* non-fatal, continue */
9484         }
9485
9486         return result;
9487 }
9488
9489 /**
9490  * i40e_pci_error_resume - restart operations after PCI error recovery
9491  * @pdev: PCI device information struct
9492  *
9493  * Called to allow the driver to bring things back up after PCI error
9494  * and/or reset recovery has finished.
9495  **/
9496 static void i40e_pci_error_resume(struct pci_dev *pdev)
9497 {
9498         struct i40e_pf *pf = pci_get_drvdata(pdev);
9499
9500         dev_info(&pdev->dev, "%s\n", __func__);
9501         if (test_bit(__I40E_SUSPENDED, &pf->state))
9502                 return;
9503
9504         rtnl_lock();
9505         i40e_handle_reset_warning(pf);
9506         rtnl_lock();
9507 }
9508
9509 /**
9510  * i40e_shutdown - PCI callback for shutting down
9511  * @pdev: PCI device information struct
9512  **/
9513 static void i40e_shutdown(struct pci_dev *pdev)
9514 {
9515         struct i40e_pf *pf = pci_get_drvdata(pdev);
9516         struct i40e_hw *hw = &pf->hw;
9517
9518         set_bit(__I40E_SUSPENDED, &pf->state);
9519         set_bit(__I40E_DOWN, &pf->state);
9520         rtnl_lock();
9521         i40e_prep_for_reset(pf);
9522         rtnl_unlock();
9523
9524         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
9525         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
9526
9527         if (system_state == SYSTEM_POWER_OFF) {
9528                 pci_wake_from_d3(pdev, pf->wol_en);
9529                 pci_set_power_state(pdev, PCI_D3hot);
9530         }
9531 }
9532
9533 #ifdef CONFIG_PM
9534 /**
9535  * i40e_suspend - PCI callback for moving to D3
9536  * @pdev: PCI device information struct
9537  **/
9538 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
9539 {
9540         struct i40e_pf *pf = pci_get_drvdata(pdev);
9541         struct i40e_hw *hw = &pf->hw;
9542
9543         set_bit(__I40E_SUSPENDED, &pf->state);
9544         set_bit(__I40E_DOWN, &pf->state);
9545         rtnl_lock();
9546         i40e_prep_for_reset(pf);
9547         rtnl_unlock();
9548
9549         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
9550         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
9551
9552         pci_wake_from_d3(pdev, pf->wol_en);
9553         pci_set_power_state(pdev, PCI_D3hot);
9554
9555         return 0;
9556 }
9557
9558 /**
9559  * i40e_resume - PCI callback for waking up from D3
9560  * @pdev: PCI device information struct
9561  **/
9562 static int i40e_resume(struct pci_dev *pdev)
9563 {
9564         struct i40e_pf *pf = pci_get_drvdata(pdev);
9565         u32 err;
9566
9567         pci_set_power_state(pdev, PCI_D0);
9568         pci_restore_state(pdev);
9569         /* pci_restore_state() clears dev->state_saves, so
9570          * call pci_save_state() again to restore it.
9571          */
9572         pci_save_state(pdev);
9573
9574         err = pci_enable_device_mem(pdev);
9575         if (err) {
9576                 dev_err(&pdev->dev,
9577                         "%s: Cannot enable PCI device from suspend\n",
9578                         __func__);
9579                 return err;
9580         }
9581         pci_set_master(pdev);
9582
9583         /* no wakeup events while running */
9584         pci_wake_from_d3(pdev, false);
9585
9586         /* handling the reset will rebuild the device state */
9587         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
9588                 clear_bit(__I40E_DOWN, &pf->state);
9589                 rtnl_lock();
9590                 i40e_reset_and_rebuild(pf, false);
9591                 rtnl_unlock();
9592         }
9593
9594         return 0;
9595 }
9596
9597 #endif
9598 static const struct pci_error_handlers i40e_err_handler = {
9599         .error_detected = i40e_pci_error_detected,
9600         .slot_reset = i40e_pci_error_slot_reset,
9601         .resume = i40e_pci_error_resume,
9602 };
9603
9604 static struct pci_driver i40e_driver = {
9605         .name     = i40e_driver_name,
9606         .id_table = i40e_pci_tbl,
9607         .probe    = i40e_probe,
9608         .remove   = i40e_remove,
9609 #ifdef CONFIG_PM
9610         .suspend  = i40e_suspend,
9611         .resume   = i40e_resume,
9612 #endif
9613         .shutdown = i40e_shutdown,
9614         .err_handler = &i40e_err_handler,
9615         .sriov_configure = i40e_pci_sriov_configure,
9616 };
9617
9618 /**
9619  * i40e_init_module - Driver registration routine
9620  *
9621  * i40e_init_module is the first routine called when the driver is
9622  * loaded. All it does is register with the PCI subsystem.
9623  **/
9624 static int __init i40e_init_module(void)
9625 {
9626         pr_info("%s: %s - version %s\n", i40e_driver_name,
9627                 i40e_driver_string, i40e_driver_version_str);
9628         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
9629         i40e_dbg_init();
9630         return pci_register_driver(&i40e_driver);
9631 }
9632 module_init(i40e_init_module);
9633
9634 /**
9635  * i40e_exit_module - Driver exit cleanup routine
9636  *
9637  * i40e_exit_module is called just before the driver is removed
9638  * from memory.
9639  **/
9640 static void __exit i40e_exit_module(void)
9641 {
9642         pci_unregister_driver(&i40e_driver);
9643         i40e_dbg_exit();
9644 }
9645 module_exit(i40e_exit_module);