drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include <linux/bpf.h>
  38 #include "en.h"
  39 #include "en_tc.h"
  40 #include "eswitch.h"
  41 #include "vxlan.h"
  42
  43 struct mlx5e_rq_param {
  44         u32                     rqc[MLX5_ST_SZ_DW(rqc)];
  45         struct mlx5_wq_param    wq;
  46         bool                    am_enabled;
  47 };
  48
  49 struct mlx5e_sq_param {
  50         u32                        sqc[MLX5_ST_SZ_DW(sqc)];
  51         struct mlx5_wq_param       wq;
  52         u16                        max_inline;
  53         u8                         min_inline_mode;
  54         bool                       icosq;
  55 };
  56
  57 struct mlx5e_cq_param {
  58         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
  59         struct mlx5_wq_param       wq;
  60         u16                        eq_ix;
  61         u8                         cq_period_mode;
  62 };
  63
  64 struct mlx5e_channel_param {
  65         struct mlx5e_rq_param      rq;
  66         struct mlx5e_sq_param      sq;
  67         struct mlx5e_sq_param      icosq;
  68         struct mlx5e_cq_param      rx_cq;
  69         struct mlx5e_cq_param      tx_cq;
  70         struct mlx5e_cq_param      icosq_cq;
  71 };
  72
  73 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  74 {
  75         return MLX5_CAP_GEN(mdev, striding_rq) &&
  76                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  77                 MLX5_CAP_ETH(mdev, reg_umr_sq);
  78 }
  79
  80 static void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
  81 {
  82         priv->params.rq_wq_type = rq_type;
  83         switch (priv->params.rq_wq_type) {
  84         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
  85                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
  86                 priv->params.mpwqe_log_stride_sz = priv->params.rx_cqe_compress ?
  87                         MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
  88                         MLX5_MPWRQ_LOG_STRIDE_SIZE;
  89                 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
  90                         priv->params.mpwqe_log_stride_sz;
  91                 break;
  92         default: /* MLX5_WQ_TYPE_LINKED_LIST */
  93                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
  94         }
  95         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
  96                                                BIT(priv->params.log_rq_size));
  97
  98         mlx5_core_info(priv->mdev,
  99                        "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
 100                        priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
 101                        BIT(priv->params.log_rq_size),
 102                        BIT(priv->params.mpwqe_log_stride_sz),
 103                        priv->params.rx_cqe_compress_admin);
 104 }
 105
 106 static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
 107 {
 108         u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) &&
 109                     !priv->xdp_prog ?
 110                     MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 111                     MLX5_WQ_TYPE_LINKED_LIST;
 112         mlx5e_set_rq_type_params(priv, rq_type);
 113 }
 114
 115 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
 116 {
 117         struct mlx5_core_dev *mdev = priv->mdev;
 118         u8 port_state;
 119
 120         port_state = mlx5_query_vport_state(mdev,
 121                 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 122
 123         if (port_state == VPORT_STATE_UP) {
 124                 netdev_info(priv->netdev, "Link up\n");
 125                 netif_carrier_on(priv->netdev);
 126         } else {
 127                 netdev_info(priv->netdev, "Link down\n");
 128                 netif_carrier_off(priv->netdev);
 129         }
 130 }
 131
 132 static void mlx5e_update_carrier_work(struct work_struct *work)
 133 {
 134         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 135                                                update_carrier_work);
 136
 137         mutex_lock(&priv->state_lock);
 138         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 139                 mlx5e_update_carrier(priv);
 140         mutex_unlock(&priv->state_lock);
 141 }
 142
 143 static void mlx5e_tx_timeout_work(struct work_struct *work)
 144 {
 145         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 146                                                tx_timeout_work);
 147         int err;
 148
 149         rtnl_lock();
 150         mutex_lock(&priv->state_lock);
 151         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 152                 goto unlock;
 153         mlx5e_close_locked(priv->netdev);
 154         err = mlx5e_open_locked(priv->netdev);
 155         if (err)
 156                 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
 157                            err);
 158 unlock:
 159         mutex_unlock(&priv->state_lock);
 160         rtnl_unlock();
 161 }
 162
 163 static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 164 {
 165         struct mlx5e_sw_stats *s = &priv->stats.sw;
 166         struct mlx5e_rq_stats *rq_stats;
 167         struct mlx5e_sq_stats *sq_stats;
 168         u64 tx_offload_none = 0;
 169         int i, j;
 170
 171         memset(s, 0, sizeof(*s));
 172         for (i = 0; i < priv->params.num_channels; i++) {
 173                 rq_stats = &priv->channel[i]->rq.stats;
 174
 175                 s->rx_packets   += rq_stats->packets;
 176                 s->rx_bytes     += rq_stats->bytes;
 177                 s->rx_lro_packets += rq_stats->lro_packets;
 178                 s->rx_lro_bytes += rq_stats->lro_bytes;
 179                 s->rx_csum_none += rq_stats->csum_none;
 180                 s->rx_csum_complete += rq_stats->csum_complete;
 181                 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
 182                 s->rx_xdp_drop += rq_stats->xdp_drop;
 183                 s->rx_wqe_err   += rq_stats->wqe_err;
 184                 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
 185                 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
 186                 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
 187                 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
 188                 s->rx_cache_reuse += rq_stats->cache_reuse;
 189                 s->rx_cache_full  += rq_stats->cache_full;
 190                 s->rx_cache_empty += rq_stats->cache_empty;
 191                 s->rx_cache_busy  += rq_stats->cache_busy;
 192
 193                 for (j = 0; j < priv->params.num_tc; j++) {
 194                         sq_stats = &priv->channel[i]->sq[j].stats;
 195
 196                         s->tx_packets           += sq_stats->packets;
 197                         s->tx_bytes             += sq_stats->bytes;
 198                         s->tx_tso_packets       += sq_stats->tso_packets;
 199                         s->tx_tso_bytes         += sq_stats->tso_bytes;
 200                         s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
 201                         s->tx_tso_inner_bytes   += sq_stats->tso_inner_bytes;
 202                         s->tx_queue_stopped     += sq_stats->stopped;
 203                         s->tx_queue_wake        += sq_stats->wake;
 204                         s->tx_queue_dropped     += sq_stats->dropped;
 205                         s->tx_xmit_more         += sq_stats->xmit_more;
 206                         s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
 207                         tx_offload_none         += sq_stats->csum_none;
 208                 }
 209         }
 210
 211         /* Update calculated offload counters */
 212         s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
 213         s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
 214
 215         s->link_down_events_phy = MLX5_GET(ppcnt_reg,
 216                                 priv->stats.pport.phy_counters,
 217                                 counter_set.phys_layer_cntrs.link_down_events);
 218 }
 219
 220 static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
 221 {
 222         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 223         u32 *out = (u32 *)priv->stats.vport.query_vport_out;
 224         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
 225         struct mlx5_core_dev *mdev = priv->mdev;
 226
 227         MLX5_SET(query_vport_counter_in, in, opcode,
 228                  MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 229         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 230         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 231
 232         memset(out, 0, outlen);
 233         mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 234 }
 235
 236 static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 237 {
 238         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
 239         struct mlx5_core_dev *mdev = priv->mdev;
 240         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 241         int prio;
 242         void *out;
 243         u32 *in;
 244
 245         in = mlx5_vzalloc(sz);
 246         if (!in)
 247                 goto free_out;
 248
 249         MLX5_SET(ppcnt_reg, in, local_port, 1);
 250
 251         out = pstats->IEEE_802_3_counters;
 252         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 253         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 254
 255         out = pstats->RFC_2863_counters;
 256         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 257         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 258
 259         out = pstats->RFC_2819_counters;
 260         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 261         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 262
 263         out = pstats->phy_counters;
 264         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 265         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 266
 267         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 268         for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 269                 out = pstats->per_prio_counters[prio];
 270                 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
 271                 mlx5_core_access_reg(mdev, in, sz, out, sz,
 272                                      MLX5_REG_PPCNT, 0, 0);
 273         }
 274
 275 free_out:
 276         kvfree(in);
 277 }
 278
 279 static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
 280 {
 281         struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
 282
 283         if (!priv->q_counter)
 284                 return;
 285
 286         mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
 287                                       &qcnt->rx_out_of_buffer);
 288 }
 289
 290 void mlx5e_update_stats(struct mlx5e_priv *priv)
 291 {
 292         mlx5e_update_q_counter(priv);
 293         mlx5e_update_vport_counters(priv);
 294         mlx5e_update_pport_counters(priv);
 295         mlx5e_update_sw_counters(priv);
 296 }
 297
 298 void mlx5e_update_stats_work(struct work_struct *work)
 299 {
 300         struct delayed_work *dwork = to_delayed_work(work);
 301         struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
 302                                                update_stats_work);
 303         mutex_lock(&priv->state_lock);
 304         if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 305                 priv->profile->update_stats(priv);
 306                 queue_delayed_work(priv->wq, dwork,
 307                                    msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
 308         }
 309         mutex_unlock(&priv->state_lock);
 310 }
 311
 312 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
 313                               enum mlx5_dev_event event, unsigned long param)
 314 {
 315         struct mlx5e_priv *priv = vpriv;
 316
 317         if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
 318                 return;
 319
 320         switch (event) {
 321         case MLX5_DEV_EVENT_PORT_UP:
 322         case MLX5_DEV_EVENT_PORT_DOWN:
 323                 queue_work(priv->wq, &priv->update_carrier_work);
 324                 break;
 325
 326         default:
 327                 break;
 328         }
 329 }
 330
 331 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 332 {
 333         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 334 }
 335
 336 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 337 {
 338         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 339         synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
 340 }
 341
 342 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 343 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 344
 345 static inline int mlx5e_get_wqe_mtt_sz(void)
 346 {
 347         /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
 348          * To avoid copying garbage after the mtt array, we allocate
 349          * a little more.
 350          */
 351         return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
 352                      MLX5_UMR_MTT_ALIGNMENT);
 353 }
 354
 355 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
 356                                        struct mlx5e_umr_wqe *wqe, u16 ix)
 357 {
 358         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 359         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 360         struct mlx5_wqe_data_seg      *dseg = &wqe->data;
 361         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 362         u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
 363         u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
 364
 365         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 366                                       ds_cnt);
 367         cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
 368         cseg->imm       = rq->mkey_be;
 369
 370         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
 371         ucseg->klm_octowords =
 372                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 373         ucseg->bsf_octowords =
 374                 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
 375         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 376
 377         dseg->lkey = sq->mkey_be;
 378         dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
 379 }
 380
 381 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
 382                                      struct mlx5e_channel *c)
 383 {
 384         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 385         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 386         int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
 387         int i;
 388
 389         rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
 390                                       GFP_KERNEL, cpu_to_node(c->cpu));
 391         if (!rq->mpwqe.info)
 392                 goto err_out;
 393
 394         /* We allocate more than mtt_sz as we will align the pointer */
 395         rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
 396                                         cpu_to_node(c->cpu));
 397         if (unlikely(!rq->mpwqe.mtt_no_align))
 398                 goto err_free_wqe_info;
 399
 400         for (i = 0; i < wq_sz; i++) {
 401                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 402
 403                 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
 404                                         MLX5_UMR_ALIGN);
 405                 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
 406                                                   PCI_DMA_TODEVICE);
 407                 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
 408                         goto err_unmap_mtts;
 409
 410                 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
 411         }
 412
 413         return 0;
 414
 415 err_unmap_mtts:
 416         while (--i >= 0) {
 417                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 418
 419                 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
 420                                  PCI_DMA_TODEVICE);
 421         }
 422         kfree(rq->mpwqe.mtt_no_align);
 423 err_free_wqe_info:
 424         kfree(rq->mpwqe.info);
 425
 426 err_out:
 427         return -ENOMEM;
 428 }
 429
 430 static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
 431 {
 432         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 433         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 434         int i;
 435
 436         for (i = 0; i < wq_sz; i++) {
 437                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 438
 439                 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
 440                                  PCI_DMA_TODEVICE);
 441         }
 442         kfree(rq->mpwqe.mtt_no_align);
 443         kfree(rq->mpwqe.info);
 444 }
 445
 446 static int mlx5e_create_rq(struct mlx5e_channel *c,
 447                            struct mlx5e_rq_param *param,
 448                            struct mlx5e_rq *rq)
 449 {
 450         struct mlx5e_priv *priv = c->priv;
 451         struct mlx5_core_dev *mdev = priv->mdev;
 452         void *rqc = param->rqc;
 453         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 454         u32 byte_count;
 455         u32 frag_sz;
 456         int npages;
 457         int wq_sz;
 458         int err;
 459         int i;
 460
 461         param->wq.db_numa_node = cpu_to_node(c->cpu);
 462
 463         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
 464                                 &rq->wq_ctrl);
 465         if (err)
 466                 return err;
 467
 468         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
 469
 470         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 471
 472         rq->wq_type = priv->params.rq_wq_type;
 473         rq->pdev    = c->pdev;
 474         rq->netdev  = c->netdev;
 475         rq->tstamp  = &priv->tstamp;
 476         rq->channel = c;
 477         rq->ix      = c->ix;
 478         rq->priv    = c->priv;
 479         rq->xdp_prog = priv->xdp_prog;
 480
 481         switch (priv->params.rq_wq_type) {
 482         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 483                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
 484                 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
 485                 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 486
 487                 rq->mpwqe.mtt_offset = c->ix *
 488                         MLX5E_REQUIRED_MTTS(1, BIT(priv->params.log_rq_size));
 489
 490                 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
 491                 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
 492
 493                 rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
 494                 byte_count = rq->buff.wqe_sz;
 495                 rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
 496                 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
 497                 if (err)
 498                         goto err_rq_wq_destroy;
 499                 break;
 500         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 501                 rq->dma_info = kzalloc_node(wq_sz * sizeof(*rq->dma_info),
 502                                             GFP_KERNEL, cpu_to_node(c->cpu));
 503                 if (!rq->dma_info) {
 504                         err = -ENOMEM;
 505                         goto err_rq_wq_destroy;
 506                 }
 507
 508                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
 509                 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
 510                 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 511
 512                 rq->buff.wqe_sz = (priv->params.lro_en) ?
 513                                 priv->params.lro_wqe_sz :
 514                                 MLX5E_SW2HW_MTU(priv->netdev->mtu);
 515                 byte_count = rq->buff.wqe_sz;
 516
 517                 /* calc the required page order */
 518                 frag_sz = MLX5_RX_HEADROOM +
 519                           byte_count /* packet data */ +
 520                           SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 521                 frag_sz = SKB_DATA_ALIGN(frag_sz);
 522
 523                 npages = DIV_ROUND_UP(frag_sz, PAGE_SIZE);
 524                 rq->buff.page_order = order_base_2(npages);
 525
 526                 byte_count |= MLX5_HW_START_PADDING;
 527                 rq->mkey_be = c->mkey_be;
 528         }
 529
 530         for (i = 0; i < wq_sz; i++) {
 531                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 532
 533                 wqe->data.byte_count = cpu_to_be32(byte_count);
 534                 wqe->data.lkey = rq->mkey_be;
 535         }
 536
 537         INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
 538         rq->am.mode = priv->params.rx_cq_period_mode;
 539
 540         rq->page_cache.head = 0;
 541         rq->page_cache.tail = 0;
 542
 543         if (rq->xdp_prog)
 544                 bpf_prog_add(rq->xdp_prog, 1);
 545
 546         return 0;
 547
 548 err_rq_wq_destroy:
 549         mlx5_wq_destroy(&rq->wq_ctrl);
 550
 551         return err;
 552 }
 553
 554 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 555 {
 556         int i;
 557
 558         if (rq->xdp_prog)
 559                 bpf_prog_put(rq->xdp_prog);
 560
 561         switch (rq->wq_type) {
 562         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 563                 mlx5e_rq_free_mpwqe_info(rq);
 564                 break;
 565         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 566                 kfree(rq->dma_info);
 567         }
 568
 569         for (i = rq->page_cache.head; i != rq->page_cache.tail;
 570              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 571                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 572
 573                 mlx5e_page_release(rq, dma_info, false);
 574         }
 575         mlx5_wq_destroy(&rq->wq_ctrl);
 576 }
 577
 578 static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 579 {
 580         struct mlx5e_priv *priv = rq->priv;
 581         struct mlx5_core_dev *mdev = priv->mdev;
 582
 583         void *in;
 584         void *rqc;
 585         void *wq;
 586         int inlen;
 587         int err;
 588
 589         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 590                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 591         in = mlx5_vzalloc(inlen);
 592         if (!in)
 593                 return -ENOMEM;
 594
 595         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 596         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 597
 598         memcpy(rqc, param->rqc, sizeof(param->rqc));
 599
 600         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 601         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 602         MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
 603         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 604                                                 MLX5_ADAPTER_PAGE_SHIFT);
 605         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 606
 607         mlx5_fill_page_array(&rq->wq_ctrl.buf,
 608                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 609
 610         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 611
 612         kvfree(in);
 613
 614         return err;
 615 }
 616
 617 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
 618                                  int next_state)
 619 {
 620         struct mlx5e_channel *c = rq->channel;
 621         struct mlx5e_priv *priv = c->priv;
 622         struct mlx5_core_dev *mdev = priv->mdev;
 623
 624         void *in;
 625         void *rqc;
 626         int inlen;
 627         int err;
 628
 629         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 630         in = mlx5_vzalloc(inlen);
 631         if (!in)
 632                 return -ENOMEM;
 633
 634         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 635
 636         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 637         MLX5_SET(rqc, rqc, state, next_state);
 638
 639         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 640
 641         kvfree(in);
 642
 643         return err;
 644 }
 645
 646 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 647 {
 648         struct mlx5e_channel *c = rq->channel;
 649         struct mlx5e_priv *priv = c->priv;
 650         struct mlx5_core_dev *mdev = priv->mdev;
 651
 652         void *in;
 653         void *rqc;
 654         int inlen;
 655         int err;
 656
 657         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 658         in = mlx5_vzalloc(inlen);
 659         if (!in)
 660                 return -ENOMEM;
 661
 662         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 663
 664         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 665         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 666                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 667         MLX5_SET(rqc, rqc, vsd, vsd);
 668         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 669
 670         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 671
 672         kvfree(in);
 673
 674         return err;
 675 }
 676
 677 static void mlx5e_disable_rq(struct mlx5e_rq *rq)
 678 {
 679         mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
 680 }
 681
 682 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
 683 {
 684         unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
 685         struct mlx5e_channel *c = rq->channel;
 686         struct mlx5e_priv *priv = c->priv;
 687         struct mlx5_wq_ll *wq = &rq->wq;
 688
 689         while (time_before(jiffies, exp_time)) {
 690                 if (wq->cur_sz >= priv->params.min_rx_wqes)
 691                         return 0;
 692
 693                 msleep(20);
 694         }
 695
 696         return -ETIMEDOUT;
 697 }
 698
 699 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 700 {
 701         struct mlx5_wq_ll *wq = &rq->wq;
 702         struct mlx5e_rx_wqe *wqe;
 703         __be16 wqe_ix_be;
 704         u16 wqe_ix;
 705
 706         /* UMR WQE (if in progress) is always at wq->head */
 707         if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
 708                 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
 709
 710         while (!mlx5_wq_ll_is_empty(wq)) {
 711                 wqe_ix_be = *wq->tail_next;
 712                 wqe_ix    = be16_to_cpu(wqe_ix_be);
 713                 wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
 714                 rq->dealloc_wqe(rq, wqe_ix);
 715                 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
 716                                &wqe->next.next_wqe_index);
 717         }
 718 }
 719
 720 static int mlx5e_open_rq(struct mlx5e_channel *c,
 721                          struct mlx5e_rq_param *param,
 722                          struct mlx5e_rq *rq)
 723 {
 724         struct mlx5e_sq *sq = &c->icosq;
 725         u16 pi = sq->pc & sq->wq.sz_m1;
 726         int err;
 727
 728         err = mlx5e_create_rq(c, param, rq);
 729         if (err)
 730                 return err;
 731
 732         err = mlx5e_enable_rq(rq, param);
 733         if (err)
 734                 goto err_destroy_rq;
 735
 736         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 737         if (err)
 738                 goto err_disable_rq;
 739
 740         if (param->am_enabled)
 741                 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 742
 743         sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;
 744         sq->ico_wqe_info[pi].num_wqebbs = 1;
 745         mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
 746
 747         return 0;
 748
 749 err_disable_rq:
 750         mlx5e_disable_rq(rq);
 751 err_destroy_rq:
 752         mlx5e_destroy_rq(rq);
 753
 754         return err;
 755 }
 756
 757 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 758 {
 759         set_bit(MLX5E_RQ_STATE_FLUSH, &rq->state);
 760         napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
 761         cancel_work_sync(&rq->am.work);
 762
 763         mlx5e_disable_rq(rq);
 764         mlx5e_free_rx_descs(rq);
 765         mlx5e_destroy_rq(rq);
 766 }
 767
 768 static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
 769 {
 770         kfree(sq->wqe_info);
 771         kfree(sq->dma_fifo);
 772         kfree(sq->skb);
 773 }
 774
 775 static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
 776 {
 777         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 778         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 779
 780         sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);
 781         sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,
 782                                     numa);
 783         sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,
 784                                     numa);
 785
 786         if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {
 787                 mlx5e_free_sq_db(sq);
 788                 return -ENOMEM;
 789         }
 790
 791         sq->dma_fifo_mask = df_sz - 1;
 792
 793         return 0;
 794 }
 795
 796 static int mlx5e_create_sq(struct mlx5e_channel *c,
 797                            int tc,
 798                            struct mlx5e_sq_param *param,
 799                            struct mlx5e_sq *sq)
 800 {
 801         struct mlx5e_priv *priv = c->priv;
 802         struct mlx5_core_dev *mdev = priv->mdev;
 803
 804         void *sqc = param->sqc;
 805         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
 806         int err;
 807
 808         err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));
 809         if (err)
 810                 return err;
 811
 812         param->wq.db_numa_node = cpu_to_node(c->cpu);
 813
 814         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
 815                                  &sq->wq_ctrl);
 816         if (err)
 817                 goto err_unmap_free_uar;
 818
 819         sq->wq.db       = &sq->wq.db[MLX5_SND_DBR];
 820         if (sq->uar.bf_map) {
 821                 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
 822                 sq->uar_map = sq->uar.bf_map;
 823         } else {
 824                 sq->uar_map = sq->uar.map;
 825         }
 826         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
 827         sq->max_inline  = param->max_inline;
 828         sq->min_inline_mode =
 829                 MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5E_INLINE_MODE_VPORT_CONTEXT ?
 830                 param->min_inline_mode : 0;
 831
 832         err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
 833         if (err)
 834                 goto err_sq_wq_destroy;
 835
 836         if (param->icosq) {
 837                 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 838
 839                 sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *
 840                                                 wq_sz,
 841                                                 GFP_KERNEL,
 842                                                 cpu_to_node(c->cpu));
 843                 if (!sq->ico_wqe_info) {
 844                         err = -ENOMEM;
 845                         goto err_free_sq_db;
 846                 }
 847         } else {
 848                 int txq_ix;
 849
 850                 txq_ix = c->ix + tc * priv->params.num_channels;
 851                 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
 852                 priv->txq_to_sq_map[txq_ix] = sq;
 853         }
 854
 855         sq->pdev      = c->pdev;
 856         sq->tstamp    = &priv->tstamp;
 857         sq->mkey_be   = c->mkey_be;
 858         sq->channel   = c;
 859         sq->tc        = tc;
 860         sq->edge      = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
 861         sq->bf_budget = MLX5E_SQ_BF_BUDGET;
 862
 863         return 0;
 864
 865 err_free_sq_db:
 866         mlx5e_free_sq_db(sq);
 867
 868 err_sq_wq_destroy:
 869         mlx5_wq_destroy(&sq->wq_ctrl);
 870
 871 err_unmap_free_uar:
 872         mlx5_unmap_free_uar(mdev, &sq->uar);
 873
 874         return err;
 875 }
 876
 877 static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
 878 {
 879         struct mlx5e_channel *c = sq->channel;
 880         struct mlx5e_priv *priv = c->priv;
 881
 882         kfree(sq->ico_wqe_info);
 883         mlx5e_free_sq_db(sq);
 884         mlx5_wq_destroy(&sq->wq_ctrl);
 885         mlx5_unmap_free_uar(priv->mdev, &sq->uar);
 886 }
 887
 888 static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
 889 {
 890         struct mlx5e_channel *c = sq->channel;
 891         struct mlx5e_priv *priv = c->priv;
 892         struct mlx5_core_dev *mdev = priv->mdev;
 893
 894         void *in;
 895         void *sqc;
 896         void *wq;
 897         int inlen;
 898         int err;
 899
 900         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
 901                 sizeof(u64) * sq->wq_ctrl.buf.npages;
 902         in = mlx5_vzalloc(inlen);
 903         if (!in)
 904                 return -ENOMEM;
 905
 906         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 907         wq = MLX5_ADDR_OF(sqc, sqc, wq);
 908
 909         memcpy(sqc, param->sqc, sizeof(param->sqc));
 910
 911         MLX5_SET(sqc,  sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);
 912         MLX5_SET(sqc,  sqc, cqn,                sq->cq.mcq.cqn);
 913         MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);
 914         MLX5_SET(sqc,  sqc, state,              MLX5_SQC_STATE_RST);
 915         MLX5_SET(sqc,  sqc, tis_lst_sz,         param->icosq ? 0 : 1);
 916         MLX5_SET(sqc,  sqc, flush_in_error_en,  1);
 917
 918         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
 919         MLX5_SET(wq,   wq, uar_page,      sq->uar.index);
 920         MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
 921                                           MLX5_ADAPTER_PAGE_SHIFT);
 922         MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
 923
 924         mlx5_fill_page_array(&sq->wq_ctrl.buf,
 925                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 926
 927         err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
 928
 929         kvfree(in);
 930
 931         return err;
 932 }
 933
 934 static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
 935                            int next_state, bool update_rl, int rl_index)
 936 {
 937         struct mlx5e_channel *c = sq->channel;
 938         struct mlx5e_priv *priv = c->priv;
 939         struct mlx5_core_dev *mdev = priv->mdev;
 940
 941         void *in;
 942         void *sqc;
 943         int inlen;
 944         int err;
 945
 946         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
 947         in = mlx5_vzalloc(inlen);
 948         if (!in)
 949                 return -ENOMEM;
 950
 951         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
 952
 953         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
 954         MLX5_SET(sqc, sqc, state, next_state);
 955         if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
 956                 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
 957                 MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
 958         }
 959
 960         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
 961
 962         kvfree(in);
 963
 964         return err;
 965 }
 966
 967 static void mlx5e_disable_sq(struct mlx5e_sq *sq)
 968 {
 969         struct mlx5e_channel *c = sq->channel;
 970         struct mlx5e_priv *priv = c->priv;
 971         struct mlx5_core_dev *mdev = priv->mdev;
 972
 973         mlx5_core_destroy_sq(mdev, sq->sqn);
 974         if (sq->rate_limit)
 975                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
 976 }
 977
 978 static int mlx5e_open_sq(struct mlx5e_channel *c,
 979                          int tc,
 980                          struct mlx5e_sq_param *param,
 981                          struct mlx5e_sq *sq)
 982 {
 983         int err;
 984
 985         err = mlx5e_create_sq(c, tc, param, sq);
 986         if (err)
 987                 return err;
 988
 989         err = mlx5e_enable_sq(sq, param);
 990         if (err)
 991                 goto err_destroy_sq;
 992
 993         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
 994                               false, 0);
 995         if (err)
 996                 goto err_disable_sq;
 997
 998         if (sq->txq) {
 999                 netdev_tx_reset_queue(sq->txq);
1000                 netif_tx_start_queue(sq->txq);
1001         }
1002
1003         return 0;
1004
1005 err_disable_sq:
1006         mlx5e_disable_sq(sq);
1007 err_destroy_sq:
1008         mlx5e_destroy_sq(sq);
1009
1010         return err;
1011 }
1012
1013 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1014 {
1015         __netif_tx_lock_bh(txq);
1016         netif_tx_stop_queue(txq);
1017         __netif_tx_unlock_bh(txq);
1018 }
1019
1020 static void mlx5e_close_sq(struct mlx5e_sq *sq)
1021 {
1022         set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
1023         /* prevent netif_tx_wake_queue */
1024         napi_synchronize(&sq->channel->napi);
1025
1026         if (sq->txq) {
1027                 netif_tx_disable_queue(sq->txq);
1028
1029                 /* last doorbell out, godspeed .. */
1030                 if (mlx5e_sq_has_room_for(sq, 1))
1031                         mlx5e_send_nop(sq, true);
1032         }
1033
1034         mlx5e_disable_sq(sq);
1035         mlx5e_free_tx_descs(sq);
1036         mlx5e_destroy_sq(sq);
1037 }
1038
1039 static int mlx5e_create_cq(struct mlx5e_channel *c,
1040                            struct mlx5e_cq_param *param,
1041                            struct mlx5e_cq *cq)
1042 {
1043         struct mlx5e_priv *priv = c->priv;
1044         struct mlx5_core_dev *mdev = priv->mdev;
1045         struct mlx5_core_cq *mcq = &cq->mcq;
1046         int eqn_not_used;
1047         unsigned int irqn;
1048         int err;
1049         u32 i;
1050
1051         param->wq.buf_numa_node = cpu_to_node(c->cpu);
1052         param->wq.db_numa_node  = cpu_to_node(c->cpu);
1053         param->eq_ix   = c->ix;
1054
1055         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1056                                &cq->wq_ctrl);
1057         if (err)
1058                 return err;
1059
1060         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1061
1062         cq->napi        = &c->napi;
1063
1064         mcq->cqe_sz     = 64;
1065         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1066         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1067         *mcq->set_ci_db = 0;
1068         *mcq->arm_db    = 0;
1069         mcq->vector     = param->eq_ix;
1070         mcq->comp       = mlx5e_completion_event;
1071         mcq->event      = mlx5e_cq_error_event;
1072         mcq->irqn       = irqn;
1073         mcq->uar        = &mdev->mlx5e_res.cq_uar;
1074
1075         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1076                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1077
1078                 cqe->op_own = 0xf1;
1079         }
1080
1081         cq->channel = c;
1082         cq->priv = priv;
1083
1084         return 0;
1085 }
1086
1087 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1088 {
1089         mlx5_wq_destroy(&cq->wq_ctrl);
1090 }
1091
1092 static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1093 {
1094         struct mlx5e_priv *priv = cq->priv;
1095         struct mlx5_core_dev *mdev = priv->mdev;
1096         struct mlx5_core_cq *mcq = &cq->mcq;
1097
1098         void *in;
1099         void *cqc;
1100         int inlen;
1101         unsigned int irqn_not_used;
1102         int eqn;
1103         int err;
1104
1105         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1106                 sizeof(u64) * cq->wq_ctrl.buf.npages;
1107         in = mlx5_vzalloc(inlen);
1108         if (!in)
1109                 return -ENOMEM;
1110
1111         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1112
1113         memcpy(cqc, param->cqc, sizeof(param->cqc));
1114
1115         mlx5_fill_page_array(&cq->wq_ctrl.buf,
1116                              (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1117
1118         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1119
1120         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1121         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1122         MLX5_SET(cqc,   cqc, uar_page,      mcq->uar->index);
1123         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1124                                             MLX5_ADAPTER_PAGE_SHIFT);
1125         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1126
1127         err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1128
1129         kvfree(in);
1130
1131         if (err)
1132                 return err;
1133
1134         mlx5e_cq_arm(cq);
1135
1136         return 0;
1137 }
1138
1139 static void mlx5e_disable_cq(struct mlx5e_cq *cq)
1140 {
1141         struct mlx5e_priv *priv = cq->priv;
1142         struct mlx5_core_dev *mdev = priv->mdev;
1143
1144         mlx5_core_destroy_cq(mdev, &cq->mcq);
1145 }
1146
1147 static int mlx5e_open_cq(struct mlx5e_channel *c,
1148                          struct mlx5e_cq_param *param,
1149                          struct mlx5e_cq *cq,
1150                          struct mlx5e_cq_moder moderation)
1151 {
1152         int err;
1153         struct mlx5e_priv *priv = c->priv;
1154         struct mlx5_core_dev *mdev = priv->mdev;
1155
1156         err = mlx5e_create_cq(c, param, cq);
1157         if (err)
1158                 return err;
1159
1160         err = mlx5e_enable_cq(cq, param);
1161         if (err)
1162                 goto err_destroy_cq;
1163
1164         if (MLX5_CAP_GEN(mdev, cq_moderation))
1165                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
1166                                                moderation.usec,
1167                                                moderation.pkts);
1168         return 0;
1169
1170 err_destroy_cq:
1171         mlx5e_destroy_cq(cq);
1172
1173         return err;
1174 }
1175
1176 static void mlx5e_close_cq(struct mlx5e_cq *cq)
1177 {
1178         mlx5e_disable_cq(cq);
1179         mlx5e_destroy_cq(cq);
1180 }
1181
1182 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1183 {
1184         return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1185 }
1186
1187 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1188                              struct mlx5e_channel_param *cparam)
1189 {
1190         struct mlx5e_priv *priv = c->priv;
1191         int err;
1192         int tc;
1193
1194         for (tc = 0; tc < c->num_tc; tc++) {
1195                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
1196                                     priv->params.tx_cq_moderation);
1197                 if (err)
1198                         goto err_close_tx_cqs;
1199         }
1200
1201         return 0;
1202
1203 err_close_tx_cqs:
1204         for (tc--; tc >= 0; tc--)
1205                 mlx5e_close_cq(&c->sq[tc].cq);
1206
1207         return err;
1208 }
1209
1210 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1211 {
1212         int tc;
1213
1214         for (tc = 0; tc < c->num_tc; tc++)
1215                 mlx5e_close_cq(&c->sq[tc].cq);
1216 }
1217
1218 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1219                           struct mlx5e_channel_param *cparam)
1220 {
1221         int err;
1222         int tc;
1223
1224         for (tc = 0; tc < c->num_tc; tc++) {
1225                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1226                 if (err)
1227                         goto err_close_sqs;
1228         }
1229
1230         return 0;
1231
1232 err_close_sqs:
1233         for (tc--; tc >= 0; tc--)
1234                 mlx5e_close_sq(&c->sq[tc]);
1235
1236         return err;
1237 }
1238
1239 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1240 {
1241         int tc;
1242
1243         for (tc = 0; tc < c->num_tc; tc++)
1244                 mlx5e_close_sq(&c->sq[tc]);
1245 }
1246
1247 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
1248 {
1249         int i;
1250
1251         for (i = 0; i < priv->profile->max_tc; i++)
1252                 priv->channeltc_to_txq_map[ix][i] =
1253                         ix + i * priv->params.num_channels;
1254 }
1255
1256 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1257                                 struct mlx5e_sq *sq, u32 rate)
1258 {
1259         struct mlx5e_priv *priv = netdev_priv(dev);
1260         struct mlx5_core_dev *mdev = priv->mdev;
1261         u16 rl_index = 0;
1262         int err;
1263
1264         if (rate == sq->rate_limit)
1265                 /* nothing to do */
1266                 return 0;
1267
1268         if (sq->rate_limit)
1269                 /* remove current rl index to free space to next ones */
1270                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1271
1272         sq->rate_limit = 0;
1273
1274         if (rate) {
1275                 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1276                 if (err) {
1277                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1278                                    rate, err);
1279                         return err;
1280                 }
1281         }
1282
1283         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1284                               MLX5_SQC_STATE_RDY, true, rl_index);
1285         if (err) {
1286                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1287                            rate, err);
1288                 /* remove the rate from the table */
1289                 if (rate)
1290                         mlx5_rl_remove_rate(mdev, rate);
1291                 return err;
1292         }
1293
1294         sq->rate_limit = rate;
1295         return 0;
1296 }
1297
1298 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1299 {
1300         struct mlx5e_priv *priv = netdev_priv(dev);
1301         struct mlx5_core_dev *mdev = priv->mdev;
1302         struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1303         int err = 0;
1304
1305         if (!mlx5_rl_is_supported(mdev)) {
1306                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1307                 return -EINVAL;
1308         }
1309
1310         /* rate is given in Mb/sec, HW config is in Kb/sec */
1311         rate = rate << 10;
1312
1313         /* Check whether rate in valid range, 0 is always valid */
1314         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1315                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1316                 return -ERANGE;
1317         }
1318
1319         mutex_lock(&priv->state_lock);
1320         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1321                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1322         if (!err)
1323                 priv->tx_rates[index] = rate;
1324         mutex_unlock(&priv->state_lock);
1325
1326         return err;
1327 }
1328
1329 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1330                               struct mlx5e_channel_param *cparam,
1331                               struct mlx5e_channel **cp)
1332 {
1333         struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
1334         struct net_device *netdev = priv->netdev;
1335         struct mlx5e_cq_moder rx_cq_profile;
1336         int cpu = mlx5e_get_cpu(priv, ix);
1337         struct mlx5e_channel *c;
1338         struct mlx5e_sq *sq;
1339         int err;
1340         int i;
1341
1342         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1343         if (!c)
1344                 return -ENOMEM;
1345
1346         c->priv     = priv;
1347         c->ix       = ix;
1348         c->cpu      = cpu;
1349         c->pdev     = &priv->mdev->pdev->dev;
1350         c->netdev   = priv->netdev;
1351         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1352         c->num_tc   = priv->params.num_tc;
1353
1354         if (priv->params.rx_am_enabled)
1355                 rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1356         else
1357                 rx_cq_profile = priv->params.rx_cq_moderation;
1358
1359         mlx5e_build_channeltc_to_txq_map(priv, ix);
1360
1361         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1362
1363         err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
1364         if (err)
1365                 goto err_napi_del;
1366
1367         err = mlx5e_open_tx_cqs(c, cparam);
1368         if (err)
1369                 goto err_close_icosq_cq;
1370
1371         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1372                             rx_cq_profile);
1373         if (err)
1374                 goto err_close_tx_cqs;
1375
1376         napi_enable(&c->napi);
1377
1378         err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
1379         if (err)
1380                 goto err_disable_napi;
1381
1382         err = mlx5e_open_sqs(c, cparam);
1383         if (err)
1384                 goto err_close_icosq;
1385
1386         for (i = 0; i < priv->params.num_tc; i++) {
1387                 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1388
1389                 if (priv->tx_rates[txq_ix]) {
1390                         sq = priv->txq_to_sq_map[txq_ix];
1391                         mlx5e_set_sq_maxrate(priv->netdev, sq,
1392                                              priv->tx_rates[txq_ix]);
1393                 }
1394         }
1395
1396         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1397         if (err)
1398                 goto err_close_sqs;
1399
1400         netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1401         *cp = c;
1402
1403         return 0;
1404
1405 err_close_sqs:
1406         mlx5e_close_sqs(c);
1407
1408 err_close_icosq:
1409         mlx5e_close_sq(&c->icosq);
1410
1411 err_disable_napi:
1412         napi_disable(&c->napi);
1413         mlx5e_close_cq(&c->rq.cq);
1414
1415 err_close_tx_cqs:
1416         mlx5e_close_tx_cqs(c);
1417
1418 err_close_icosq_cq:
1419         mlx5e_close_cq(&c->icosq.cq);
1420
1421 err_napi_del:
1422         netif_napi_del(&c->napi);
1423         napi_hash_del(&c->napi);
1424         kfree(c);
1425
1426         return err;
1427 }
1428
1429 static void mlx5e_close_channel(struct mlx5e_channel *c)
1430 {
1431         mlx5e_close_rq(&c->rq);
1432         mlx5e_close_sqs(c);
1433         mlx5e_close_sq(&c->icosq);
1434         napi_disable(&c->napi);
1435         mlx5e_close_cq(&c->rq.cq);
1436         mlx5e_close_tx_cqs(c);
1437         mlx5e_close_cq(&c->icosq.cq);
1438         netif_napi_del(&c->napi);
1439
1440         napi_hash_del(&c->napi);
1441         synchronize_rcu();
1442
1443         kfree(c);
1444 }
1445
1446 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1447                                  struct mlx5e_rq_param *param)
1448 {
1449         void *rqc = param->rqc;
1450         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1451
1452         switch (priv->params.rq_wq_type) {
1453         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1454                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1455                          priv->params.mpwqe_log_num_strides - 9);
1456                 MLX5_SET(wq, wq, log_wqe_stride_size,
1457                          priv->params.mpwqe_log_stride_sz - 6);
1458                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1459                 break;
1460         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1461                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1462         }
1463
1464         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1465         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1466         MLX5_SET(wq, wq, log_wq_sz,        priv->params.log_rq_size);
1467         MLX5_SET(wq, wq, pd,               priv->mdev->mlx5e_res.pdn);
1468         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1469
1470         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1471         param->wq.linear = 1;
1472
1473         param->am_enabled = priv->params.rx_am_enabled;
1474 }
1475
1476 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1477 {
1478         void *rqc = param->rqc;
1479         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1480
1481         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1482         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1483 }
1484
1485 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1486                                         struct mlx5e_sq_param *param)
1487 {
1488         void *sqc = param->sqc;
1489         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1490
1491         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1492         MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
1493
1494         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1495 }
1496
1497 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1498                                  struct mlx5e_sq_param *param)
1499 {
1500         void *sqc = param->sqc;
1501         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1502
1503         mlx5e_build_sq_param_common(priv, param);
1504         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1505
1506         param->max_inline = priv->params.tx_max_inline;
1507         param->min_inline_mode = priv->params.tx_min_inline_mode;
1508 }
1509
1510 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1511                                         struct mlx5e_cq_param *param)
1512 {
1513         void *cqc = param->cqc;
1514
1515         MLX5_SET(cqc, cqc, uar_page, priv->mdev->mlx5e_res.cq_uar.index);
1516 }
1517
1518 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1519                                     struct mlx5e_cq_param *param)
1520 {
1521         void *cqc = param->cqc;
1522         u8 log_cq_size;
1523
1524         switch (priv->params.rq_wq_type) {
1525         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1526                 log_cq_size = priv->params.log_rq_size +
1527                         priv->params.mpwqe_log_num_strides;
1528                 break;
1529         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1530                 log_cq_size = priv->params.log_rq_size;
1531         }
1532
1533         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
1534         if (priv->params.rx_cqe_compress) {
1535                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1536                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1537         }
1538
1539         mlx5e_build_common_cq_param(priv, param);
1540
1541         param->cq_period_mode = priv->params.rx_cq_period_mode;
1542 }
1543
1544 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1545                                     struct mlx5e_cq_param *param)
1546 {
1547         void *cqc = param->cqc;
1548
1549         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1550
1551         mlx5e_build_common_cq_param(priv, param);
1552
1553         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1554 }
1555
1556 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1557                                      struct mlx5e_cq_param *param,
1558                                      u8 log_wq_size)
1559 {
1560         void *cqc = param->cqc;
1561
1562         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1563
1564         mlx5e_build_common_cq_param(priv, param);
1565
1566         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1567 }
1568
1569 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1570                                     struct mlx5e_sq_param *param,
1571                                     u8 log_wq_size)
1572 {
1573         void *sqc = param->sqc;
1574         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1575
1576         mlx5e_build_sq_param_common(priv, param);
1577
1578         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1579         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
1580
1581         param->icosq = true;
1582 }
1583
1584 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
1585 {
1586         u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1587
1588         mlx5e_build_rq_param(priv, &cparam->rq);
1589         mlx5e_build_sq_param(priv, &cparam->sq);
1590         mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
1591         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1592         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1593         mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
1594 }
1595
1596 static int mlx5e_open_channels(struct mlx5e_priv *priv)
1597 {
1598         struct mlx5e_channel_param *cparam;
1599         int nch = priv->params.num_channels;
1600         int err = -ENOMEM;
1601         int i;
1602         int j;
1603
1604         priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1605                                 GFP_KERNEL);
1606
1607         priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
1608                                       sizeof(struct mlx5e_sq *), GFP_KERNEL);
1609
1610         cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1611
1612         if (!priv->channel || !priv->txq_to_sq_map || !cparam)
1613                 goto err_free_txq_to_sq_map;
1614
1615         mlx5e_build_channel_param(priv, cparam);
1616
1617         for (i = 0; i < nch; i++) {
1618                 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
1619                 if (err)
1620                         goto err_close_channels;
1621         }
1622
1623         for (j = 0; j < nch; j++) {
1624                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1625                 if (err)
1626                         goto err_close_channels;
1627         }
1628
1629         /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1630          * polling for inactive tx queues.
1631          */
1632         netif_tx_start_all_queues(priv->netdev);
1633
1634         kfree(cparam);
1635         return 0;
1636
1637 err_close_channels:
1638         for (i--; i >= 0; i--)
1639                 mlx5e_close_channel(priv->channel[i]);
1640
1641 err_free_txq_to_sq_map:
1642         kfree(priv->txq_to_sq_map);
1643         kfree(priv->channel);
1644         kfree(cparam);
1645
1646         return err;
1647 }
1648
1649 static void mlx5e_close_channels(struct mlx5e_priv *priv)
1650 {
1651         int i;
1652
1653         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1654          * polling for inactive tx queues.
1655          */
1656         netif_tx_stop_all_queues(priv->netdev);
1657         netif_tx_disable(priv->netdev);
1658
1659         for (i = 0; i < priv->params.num_channels; i++)
1660                 mlx5e_close_channel(priv->channel[i]);
1661
1662         kfree(priv->txq_to_sq_map);
1663         kfree(priv->channel);
1664 }
1665
1666 static int mlx5e_rx_hash_fn(int hfunc)
1667 {
1668         return (hfunc == ETH_RSS_HASH_TOP) ?
1669                MLX5_RX_HASH_FN_TOEPLITZ :
1670                MLX5_RX_HASH_FN_INVERTED_XOR8;
1671 }
1672
1673 static int mlx5e_bits_invert(unsigned long a, int size)
1674 {
1675         int inv = 0;
1676         int i;
1677
1678         for (i = 0; i < size; i++)
1679                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1680
1681         return inv;
1682 }
1683
1684 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1685 {
1686         int i;
1687
1688         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1689                 int ix = i;
1690                 u32 rqn;
1691
1692                 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1693                         ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1694
1695                 ix = priv->params.indirection_rqt[ix];
1696                 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1697                                 priv->channel[ix]->rq.rqn :
1698                                 priv->drop_rq.rqn;
1699                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
1700         }
1701 }
1702
1703 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1704                                       int ix)
1705 {
1706         u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1707                         priv->channel[ix]->rq.rqn :
1708                         priv->drop_rq.rqn;
1709
1710         MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
1711 }
1712
1713 static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
1714                             int ix, struct mlx5e_rqt *rqt)
1715 {
1716         struct mlx5_core_dev *mdev = priv->mdev;
1717         void *rqtc;
1718         int inlen;
1719         int err;
1720         u32 *in;
1721
1722         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1723         in = mlx5_vzalloc(inlen);
1724         if (!in)
1725                 return -ENOMEM;
1726
1727         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1728
1729         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1730         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1731
1732         if (sz > 1) /* RSS */
1733                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1734         else
1735                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1736
1737         err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
1738         if (!err)
1739                 rqt->enabled = true;
1740
1741         kvfree(in);
1742         return err;
1743 }
1744
1745 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1746 {
1747         rqt->enabled = false;
1748         mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1749 }
1750
1751 static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
1752 {
1753         struct mlx5e_rqt *rqt = &priv->indir_rqt;
1754
1755         return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
1756 }
1757
1758 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1759 {
1760         struct mlx5e_rqt *rqt;
1761         int err;
1762         int ix;
1763
1764         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1765                 rqt = &priv->direct_tir[ix].rqt;
1766                 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
1767                 if (err)
1768                         goto err_destroy_rqts;
1769         }
1770
1771         return 0;
1772
1773 err_destroy_rqts:
1774         for (ix--; ix >= 0; ix--)
1775                 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1776
1777         return err;
1778 }
1779
1780 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
1781 {
1782         struct mlx5_core_dev *mdev = priv->mdev;
1783         void *rqtc;
1784         int inlen;
1785         u32 *in;
1786         int err;
1787
1788         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1789         in = mlx5_vzalloc(inlen);
1790         if (!in)
1791                 return -ENOMEM;
1792
1793         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1794
1795         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1796         if (sz > 1) /* RSS */
1797                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1798         else
1799                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1800
1801         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1802
1803         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
1804
1805         kvfree(in);
1806
1807         return err;
1808 }
1809
1810 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1811 {
1812         u32 rqtn;
1813         int ix;
1814
1815         if (priv->indir_rqt.enabled) {
1816                 rqtn = priv->indir_rqt.rqtn;
1817                 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
1818         }
1819
1820         for (ix = 0; ix < priv->params.num_channels; ix++) {
1821                 if (!priv->direct_tir[ix].rqt.enabled)
1822                         continue;
1823                 rqtn = priv->direct_tir[ix].rqt.rqtn;
1824                 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
1825         }
1826 }
1827
1828 static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
1829 {
1830         if (!priv->params.lro_en)
1831                 return;
1832
1833 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1834
1835         MLX5_SET(tirc, tirc, lro_enable_mask,
1836                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1837                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1838         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
1839                  (priv->params.lro_wqe_sz -
1840                   ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1841         MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1842                  MLX5_CAP_ETH(priv->mdev,
1843                               lro_timer_supported_periods[2]));
1844 }
1845
1846 void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
1847 {
1848         MLX5_SET(tirc, tirc, rx_hash_fn,
1849                  mlx5e_rx_hash_fn(priv->params.rss_hfunc));
1850         if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
1851                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
1852                                              rx_hash_toeplitz_key);
1853                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
1854                                                rx_hash_toeplitz_key);
1855
1856                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1857                 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
1858         }
1859 }
1860
1861 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
1862 {
1863         struct mlx5_core_dev *mdev = priv->mdev;
1864
1865         void *in;
1866         void *tirc;
1867         int inlen;
1868         int err;
1869         int tt;
1870         int ix;
1871
1872         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1873         in = mlx5_vzalloc(inlen);
1874         if (!in)
1875                 return -ENOMEM;
1876
1877         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
1878         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
1879
1880         mlx5e_build_tir_ctx_lro(tirc, priv);
1881
1882         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
1883                 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1884                                            inlen);
1885                 if (err)
1886                         goto free_in;
1887         }
1888
1889         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1890                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
1891                                            in, inlen);
1892                 if (err)
1893                         goto free_in;
1894         }
1895
1896 free_in:
1897         kvfree(in);
1898
1899         return err;
1900 }
1901
1902 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
1903 {
1904         struct mlx5_core_dev *mdev = priv->mdev;
1905         u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
1906         int err;
1907
1908         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
1909         if (err)
1910                 return err;
1911
1912         /* Update vport context MTU */
1913         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
1914         return 0;
1915 }
1916
1917 static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
1918 {
1919         struct mlx5_core_dev *mdev = priv->mdev;
1920         u16 hw_mtu = 0;
1921         int err;
1922
1923         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
1924         if (err || !hw_mtu) /* fallback to port oper mtu */
1925                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
1926
1927         *mtu = MLX5E_HW2SW_MTU(hw_mtu);
1928 }
1929
1930 static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
1931 {
1932         struct mlx5e_priv *priv = netdev_priv(netdev);
1933         u16 mtu;
1934         int err;
1935
1936         err = mlx5e_set_mtu(priv, netdev->mtu);
1937         if (err)
1938                 return err;
1939
1940         mlx5e_query_mtu(priv, &mtu);
1941         if (mtu != netdev->mtu)
1942                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
1943                             __func__, mtu, netdev->mtu);
1944
1945         netdev->mtu = mtu;
1946         return 0;
1947 }
1948
1949 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
1950 {
1951         struct mlx5e_priv *priv = netdev_priv(netdev);
1952         int nch = priv->params.num_channels;
1953         int ntc = priv->params.num_tc;
1954         int tc;
1955
1956         netdev_reset_tc(netdev);
1957
1958         if (ntc == 1)
1959                 return;
1960
1961         netdev_set_num_tc(netdev, ntc);
1962
1963         /* Map netdev TCs to offset 0
1964          * We have our own UP to TXQ mapping for QoS
1965          */
1966         for (tc = 0; tc < ntc; tc++)
1967                 netdev_set_tc_queue(netdev, tc, nch, 0);
1968 }
1969
1970 int mlx5e_open_locked(struct net_device *netdev)
1971 {
1972         struct mlx5e_priv *priv = netdev_priv(netdev);
1973         struct mlx5_core_dev *mdev = priv->mdev;
1974         int num_txqs;
1975         int err;
1976
1977         set_bit(MLX5E_STATE_OPENED, &priv->state);
1978
1979         mlx5e_netdev_set_tcs(netdev);
1980
1981         num_txqs = priv->params.num_channels * priv->params.num_tc;
1982         netif_set_real_num_tx_queues(netdev, num_txqs);
1983         netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
1984
1985         err = mlx5e_open_channels(priv);
1986         if (err) {
1987                 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
1988                            __func__, err);
1989                 goto err_clear_state_opened_flag;
1990         }
1991
1992         err = mlx5e_refresh_tirs_self_loopback_enable(priv->mdev);
1993         if (err) {
1994                 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
1995                            __func__, err);
1996                 goto err_close_channels;
1997         }
1998
1999         mlx5e_redirect_rqts(priv);
2000         mlx5e_update_carrier(priv);
2001         mlx5e_timestamp_init(priv);
2002 #ifdef CONFIG_RFS_ACCEL
2003         priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
2004 #endif
2005         if (priv->profile->update_stats)
2006                 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
2007
2008         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
2009                 err = mlx5e_add_sqs_fwd_rules(priv);
2010                 if (err)
2011                         goto err_close_channels;
2012         }
2013         return 0;
2014
2015 err_close_channels:
2016         mlx5e_close_channels(priv);
2017 err_clear_state_opened_flag:
2018         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2019         return err;
2020 }
2021
2022 int mlx5e_open(struct net_device *netdev)
2023 {
2024         struct mlx5e_priv *priv = netdev_priv(netdev);
2025         int err;
2026
2027         mutex_lock(&priv->state_lock);
2028         err = mlx5e_open_locked(netdev);
2029         mutex_unlock(&priv->state_lock);
2030
2031         return err;
2032 }
2033
2034 int mlx5e_close_locked(struct net_device *netdev)
2035 {
2036         struct mlx5e_priv *priv = netdev_priv(netdev);
2037         struct mlx5_core_dev *mdev = priv->mdev;
2038
2039         /* May already be CLOSED in case a previous configuration operation
2040          * (e.g RX/TX queue size change) that involves close&open failed.
2041          */
2042         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2043                 return 0;
2044
2045         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2046
2047         if (MLX5_CAP_GEN(mdev, vport_group_manager))
2048                 mlx5e_remove_sqs_fwd_rules(priv);
2049
2050         mlx5e_timestamp_cleanup(priv);
2051         netif_carrier_off(priv->netdev);
2052         mlx5e_redirect_rqts(priv);
2053         mlx5e_close_channels(priv);
2054
2055         return 0;
2056 }
2057
2058 int mlx5e_close(struct net_device *netdev)
2059 {
2060         struct mlx5e_priv *priv = netdev_priv(netdev);
2061         int err;
2062
2063         if (!netif_device_present(netdev))
2064                 return -ENODEV;
2065
2066         mutex_lock(&priv->state_lock);
2067         err = mlx5e_close_locked(netdev);
2068         mutex_unlock(&priv->state_lock);
2069
2070         return err;
2071 }
2072
2073 static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
2074                                 struct mlx5e_rq *rq,
2075                                 struct mlx5e_rq_param *param)
2076 {
2077         struct mlx5_core_dev *mdev = priv->mdev;
2078         void *rqc = param->rqc;
2079         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2080         int err;
2081
2082         param->wq.db_numa_node = param->wq.buf_numa_node;
2083
2084         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2085                                 &rq->wq_ctrl);
2086         if (err)
2087                 return err;
2088
2089         rq->priv = priv;
2090
2091         return 0;
2092 }
2093
2094 static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2095                                 struct mlx5e_cq *cq,
2096                                 struct mlx5e_cq_param *param)
2097 {
2098         struct mlx5_core_dev *mdev = priv->mdev;
2099         struct mlx5_core_cq *mcq = &cq->mcq;
2100         int eqn_not_used;
2101         unsigned int irqn;
2102         int err;
2103
2104         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2105                                &cq->wq_ctrl);
2106         if (err)
2107                 return err;
2108
2109         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
2110
2111         mcq->cqe_sz     = 64;
2112         mcq->set_ci_db  = cq->wq_ctrl.db.db;
2113         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
2114         *mcq->set_ci_db = 0;
2115         *mcq->arm_db    = 0;
2116         mcq->vector     = param->eq_ix;
2117         mcq->comp       = mlx5e_completion_event;
2118         mcq->event      = mlx5e_cq_error_event;
2119         mcq->irqn       = irqn;
2120         mcq->uar        = &mdev->mlx5e_res.cq_uar;
2121
2122         cq->priv = priv;
2123
2124         return 0;
2125 }
2126
2127 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
2128 {
2129         struct mlx5e_cq_param cq_param;
2130         struct mlx5e_rq_param rq_param;
2131         struct mlx5e_rq *rq = &priv->drop_rq;
2132         struct mlx5e_cq *cq = &priv->drop_rq.cq;
2133         int err;
2134
2135         memset(&cq_param, 0, sizeof(cq_param));
2136         memset(&rq_param, 0, sizeof(rq_param));
2137         mlx5e_build_drop_rq_param(&rq_param);
2138
2139         err = mlx5e_create_drop_cq(priv, cq, &cq_param);
2140         if (err)
2141                 return err;
2142
2143         err = mlx5e_enable_cq(cq, &cq_param);
2144         if (err)
2145                 goto err_destroy_cq;
2146
2147         err = mlx5e_create_drop_rq(priv, rq, &rq_param);
2148         if (err)
2149                 goto err_disable_cq;
2150
2151         err = mlx5e_enable_rq(rq, &rq_param);
2152         if (err)
2153                 goto err_destroy_rq;
2154
2155         return 0;
2156
2157 err_destroy_rq:
2158         mlx5e_destroy_rq(&priv->drop_rq);
2159
2160 err_disable_cq:
2161         mlx5e_disable_cq(&priv->drop_rq.cq);
2162
2163 err_destroy_cq:
2164         mlx5e_destroy_cq(&priv->drop_rq.cq);
2165
2166         return err;
2167 }
2168
2169 static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
2170 {
2171         mlx5e_disable_rq(&priv->drop_rq);
2172         mlx5e_destroy_rq(&priv->drop_rq);
2173         mlx5e_disable_cq(&priv->drop_rq.cq);
2174         mlx5e_destroy_cq(&priv->drop_rq.cq);
2175 }
2176
2177 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
2178 {
2179         struct mlx5_core_dev *mdev = priv->mdev;
2180         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
2181         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2182
2183         MLX5_SET(tisc, tisc, prio, tc << 1);
2184         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
2185
2186         if (mlx5_lag_is_lacp_owner(mdev))
2187                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2188
2189         return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
2190 }
2191
2192 static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
2193 {
2194         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2195 }
2196
2197 int mlx5e_create_tises(struct mlx5e_priv *priv)
2198 {
2199         int err;
2200         int tc;
2201
2202         for (tc = 0; tc < priv->profile->max_tc; tc++) {
2203                 err = mlx5e_create_tis(priv, tc);
2204                 if (err)
2205                         goto err_close_tises;
2206         }
2207
2208         return 0;
2209
2210 err_close_tises:
2211         for (tc--; tc >= 0; tc--)
2212                 mlx5e_destroy_tis(priv, tc);
2213
2214         return err;
2215 }
2216
2217 void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
2218 {
2219         int tc;
2220
2221         for (tc = 0; tc < priv->profile->max_tc; tc++)
2222                 mlx5e_destroy_tis(priv, tc);
2223 }
2224
2225 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2226                                       enum mlx5e_traffic_types tt)
2227 {
2228         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2229
2230         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2231
2232 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2233                                  MLX5_HASH_FIELD_SEL_DST_IP)
2234
2235 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2236                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2237                                  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2238                                  MLX5_HASH_FIELD_SEL_L4_DPORT)
2239
2240 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2241                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2242                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2243
2244         mlx5e_build_tir_ctx_lro(tirc, priv);
2245
2246         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2247         MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2248         mlx5e_build_tir_ctx_hash(tirc, priv);
2249
2250         switch (tt) {
2251         case MLX5E_TT_IPV4_TCP:
2252                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2253                          MLX5_L3_PROT_TYPE_IPV4);
2254                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2255                          MLX5_L4_PROT_TYPE_TCP);
2256                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2257                          MLX5_HASH_IP_L4PORTS);
2258                 break;
2259
2260         case MLX5E_TT_IPV6_TCP:
2261                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2262                          MLX5_L3_PROT_TYPE_IPV6);
2263                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2264                          MLX5_L4_PROT_TYPE_TCP);
2265                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2266                          MLX5_HASH_IP_L4PORTS);
2267                 break;
2268
2269         case MLX5E_TT_IPV4_UDP:
2270                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2271                          MLX5_L3_PROT_TYPE_IPV4);
2272                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2273                          MLX5_L4_PROT_TYPE_UDP);
2274                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2275                          MLX5_HASH_IP_L4PORTS);
2276                 break;
2277
2278         case MLX5E_TT_IPV6_UDP:
2279                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2280                          MLX5_L3_PROT_TYPE_IPV6);
2281                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2282                          MLX5_L4_PROT_TYPE_UDP);
2283                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2284                          MLX5_HASH_IP_L4PORTS);
2285                 break;
2286
2287         case MLX5E_TT_IPV4_IPSEC_AH:
2288                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2289                          MLX5_L3_PROT_TYPE_IPV4);
2290                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2291                          MLX5_HASH_IP_IPSEC_SPI);
2292                 break;
2293
2294         case MLX5E_TT_IPV6_IPSEC_AH:
2295                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2296                          MLX5_L3_PROT_TYPE_IPV6);
2297                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2298                          MLX5_HASH_IP_IPSEC_SPI);
2299                 break;
2300
2301         case MLX5E_TT_IPV4_IPSEC_ESP:
2302                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2303                          MLX5_L3_PROT_TYPE_IPV4);
2304                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2305                          MLX5_HASH_IP_IPSEC_SPI);
2306                 break;
2307
2308         case MLX5E_TT_IPV6_IPSEC_ESP:
2309                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2310                          MLX5_L3_PROT_TYPE_IPV6);
2311                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2312                          MLX5_HASH_IP_IPSEC_SPI);
2313                 break;
2314
2315         case MLX5E_TT_IPV4:
2316                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2317                          MLX5_L3_PROT_TYPE_IPV4);
2318                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2319                          MLX5_HASH_IP);
2320                 break;
2321
2322         case MLX5E_TT_IPV6:
2323                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2324                          MLX5_L3_PROT_TYPE_IPV6);
2325                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2326                          MLX5_HASH_IP);
2327                 break;
2328         default:
2329                 WARN_ONCE(true,
2330                           "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2331         }
2332 }
2333
2334 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2335                                        u32 rqtn)
2336 {
2337         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2338
2339         mlx5e_build_tir_ctx_lro(tirc, priv);
2340
2341         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2342         MLX5_SET(tirc, tirc, indirect_table, rqtn);
2343         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2344 }
2345
2346 static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
2347 {
2348         struct mlx5e_tir *tir;
2349         void *tirc;
2350         int inlen;
2351         int err;
2352         u32 *in;
2353         int tt;
2354
2355         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2356         in = mlx5_vzalloc(inlen);
2357         if (!in)
2358                 return -ENOMEM;
2359
2360         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2361                 memset(in, 0, inlen);
2362                 tir = &priv->indir_tir[tt];
2363                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2364                 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
2365                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2366                 if (err)
2367                         goto err_destroy_tirs;
2368         }
2369
2370         kvfree(in);
2371
2372         return 0;
2373
2374 err_destroy_tirs:
2375         for (tt--; tt >= 0; tt--)
2376                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2377
2378         kvfree(in);
2379
2380         return err;
2381 }
2382
2383 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
2384 {
2385         int nch = priv->profile->max_nch(priv->mdev);
2386         struct mlx5e_tir *tir;
2387         void *tirc;
2388         int inlen;
2389         int err;
2390         u32 *in;
2391         int ix;
2392
2393         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2394         in = mlx5_vzalloc(inlen);
2395         if (!in)
2396                 return -ENOMEM;
2397
2398         for (ix = 0; ix < nch; ix++) {
2399                 memset(in, 0, inlen);
2400                 tir = &priv->direct_tir[ix];
2401                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2402                 mlx5e_build_direct_tir_ctx(priv, tirc,
2403                                            priv->direct_tir[ix].rqt.rqtn);
2404                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2405                 if (err)
2406                         goto err_destroy_ch_tirs;
2407         }
2408
2409         kvfree(in);
2410
2411         return 0;
2412
2413 err_destroy_ch_tirs:
2414         for (ix--; ix >= 0; ix--)
2415                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
2416
2417         kvfree(in);
2418
2419         return err;
2420 }
2421
2422 static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
2423 {
2424         int i;
2425
2426         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
2427                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
2428 }
2429
2430 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
2431 {
2432         int nch = priv->profile->max_nch(priv->mdev);
2433         int i;
2434
2435         for (i = 0; i < nch; i++)
2436                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2437 }
2438
2439 int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2440 {
2441         int err = 0;
2442         int i;
2443
2444         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2445                 return 0;
2446
2447         for (i = 0; i < priv->params.num_channels; i++) {
2448                 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2449                 if (err)
2450                         return err;
2451         }
2452
2453         return 0;
2454 }
2455
2456 static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2457 {
2458         struct mlx5e_priv *priv = netdev_priv(netdev);
2459         bool was_opened;
2460         int err = 0;
2461
2462         if (tc && tc != MLX5E_MAX_NUM_TC)
2463                 return -EINVAL;
2464
2465         mutex_lock(&priv->state_lock);
2466
2467         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2468         if (was_opened)
2469                 mlx5e_close_locked(priv->netdev);
2470
2471         priv->params.num_tc = tc ? tc : 1;
2472
2473         if (was_opened)
2474                 err = mlx5e_open_locked(priv->netdev);
2475
2476         mutex_unlock(&priv->state_lock);
2477
2478         return err;
2479 }
2480
2481 static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2482                               __be16 proto, struct tc_to_netdev *tc)
2483 {
2484         struct mlx5e_priv *priv = netdev_priv(dev);
2485
2486         if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2487                 goto mqprio;
2488
2489         switch (tc->type) {
2490         case TC_SETUP_CLSFLOWER:
2491                 switch (tc->cls_flower->command) {
2492                 case TC_CLSFLOWER_REPLACE:
2493                         return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2494                 case TC_CLSFLOWER_DESTROY:
2495                         return mlx5e_delete_flower(priv, tc->cls_flower);
2496                 case TC_CLSFLOWER_STATS:
2497                         return mlx5e_stats_flower(priv, tc->cls_flower);
2498                 }
2499         default:
2500                 return -EOPNOTSUPP;
2501         }
2502
2503 mqprio:
2504         if (tc->type != TC_SETUP_MQPRIO)
2505                 return -EINVAL;
2506
2507         return mlx5e_setup_tc(dev, tc->tc);
2508 }
2509
2510 struct rtnl_link_stats64 *
2511 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2512 {
2513         struct mlx5e_priv *priv = netdev_priv(dev);
2514         struct mlx5e_sw_stats *sstats = &priv->stats.sw;
2515         struct mlx5e_vport_stats *vstats = &priv->stats.vport;
2516         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
2517
2518         stats->rx_packets = sstats->rx_packets;
2519         stats->rx_bytes   = sstats->rx_bytes;
2520         stats->tx_packets = sstats->tx_packets;
2521         stats->tx_bytes   = sstats->tx_bytes;
2522
2523         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
2524         stats->tx_dropped = sstats->tx_queue_dropped;
2525
2526         stats->rx_length_errors =
2527                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2528                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2529                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
2530         stats->rx_crc_errors =
2531                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2532         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2533         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
2534         stats->tx_carrier_errors =
2535                 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
2536         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2537                            stats->rx_frame_errors;
2538         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2539
2540         /* vport multicast also counts packets that are dropped due to steering
2541          * or rx out of buffer
2542          */
2543         stats->multicast =
2544                 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
2545
2546         return stats;
2547 }
2548
2549 static void mlx5e_set_rx_mode(struct net_device *dev)
2550 {
2551         struct mlx5e_priv *priv = netdev_priv(dev);
2552
2553         queue_work(priv->wq, &priv->set_rx_mode_work);
2554 }
2555
2556 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2557 {
2558         struct mlx5e_priv *priv = netdev_priv(netdev);
2559         struct sockaddr *saddr = addr;
2560
2561         if (!is_valid_ether_addr(saddr->sa_data))
2562                 return -EADDRNOTAVAIL;
2563
2564         netif_addr_lock_bh(netdev);
2565         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2566         netif_addr_unlock_bh(netdev);
2567
2568         queue_work(priv->wq, &priv->set_rx_mode_work);
2569
2570         return 0;
2571 }
2572
2573 #define MLX5E_SET_FEATURE(netdev, feature, enable)      \
2574         do {                                            \
2575                 if (enable)                             \
2576                         netdev->features |= feature;    \
2577                 else                                    \
2578                         netdev->features &= ~feature;   \
2579         } while (0)
2580
2581 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2582
2583 static int set_feature_lro(struct net_device *netdev, bool enable)
2584 {
2585         struct mlx5e_priv *priv = netdev_priv(netdev);
2586         bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2587         int err;
2588
2589         mutex_lock(&priv->state_lock);
2590
2591         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2592                 mlx5e_close_locked(priv->netdev);
2593
2594         priv->params.lro_en = enable;
2595         err = mlx5e_modify_tirs_lro(priv);
2596         if (err) {
2597                 netdev_err(netdev, "lro modify failed, %d\n", err);
2598                 priv->params.lro_en = !enable;
2599         }
2600
2601         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2602                 mlx5e_open_locked(priv->netdev);
2603
2604         mutex_unlock(&priv->state_lock);
2605
2606         return err;
2607 }
2608
2609 static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2610 {
2611         struct mlx5e_priv *priv = netdev_priv(netdev);
2612
2613         if (enable)
2614                 mlx5e_enable_vlan_filter(priv);
2615         else
2616                 mlx5e_disable_vlan_filter(priv);
2617
2618         return 0;
2619 }
2620
2621 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2622 {
2623         struct mlx5e_priv *priv = netdev_priv(netdev);
2624
2625         if (!enable && mlx5e_tc_num_filters(priv)) {
2626                 netdev_err(netdev,
2627                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2628                 return -EINVAL;
2629         }
2630
2631         return 0;
2632 }
2633
2634 static int set_feature_rx_all(struct net_device *netdev, bool enable)
2635 {
2636         struct mlx5e_priv *priv = netdev_priv(netdev);
2637         struct mlx5_core_dev *mdev = priv->mdev;
2638
2639         return mlx5_set_port_fcs(mdev, !enable);
2640 }
2641
2642 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2643 {
2644         struct mlx5e_priv *priv = netdev_priv(netdev);
2645         int err;
2646
2647         mutex_lock(&priv->state_lock);
2648
2649         priv->params.vlan_strip_disable = !enable;
2650         err = mlx5e_modify_rqs_vsd(priv, !enable);
2651         if (err)
2652                 priv->params.vlan_strip_disable = enable;
2653
2654         mutex_unlock(&priv->state_lock);
2655
2656         return err;
2657 }
2658
2659 #ifdef CONFIG_RFS_ACCEL
2660 static int set_feature_arfs(struct net_device *netdev, bool enable)
2661 {
2662         struct mlx5e_priv *priv = netdev_priv(netdev);
2663         int err;
2664
2665         if (enable)
2666                 err = mlx5e_arfs_enable(priv);
2667         else
2668                 err = mlx5e_arfs_disable(priv);
2669
2670         return err;
2671 }
2672 #endif
2673
2674 static int mlx5e_handle_feature(struct net_device *netdev,
2675                                 netdev_features_t wanted_features,
2676                                 netdev_features_t feature,
2677                                 mlx5e_feature_handler feature_handler)
2678 {
2679         netdev_features_t changes = wanted_features ^ netdev->features;
2680         bool enable = !!(wanted_features & feature);
2681         int err;
2682
2683         if (!(changes & feature))
2684                 return 0;
2685
2686         err = feature_handler(netdev, enable);
2687         if (err) {
2688                 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2689                            enable ? "Enable" : "Disable", feature, err);
2690                 return err;
2691         }
2692
2693         MLX5E_SET_FEATURE(netdev, feature, enable);
2694         return 0;
2695 }
2696
2697 static int mlx5e_set_features(struct net_device *netdev,
2698                               netdev_features_t features)
2699 {
2700         int err;
2701
2702         err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2703                                     set_feature_lro);
2704         err |= mlx5e_handle_feature(netdev, features,
2705                                     NETIF_F_HW_VLAN_CTAG_FILTER,
2706                                     set_feature_vlan_filter);
2707         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2708                                     set_feature_tc_num_filters);
2709         err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2710                                     set_feature_rx_all);
2711         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2712                                     set_feature_rx_vlan);
2713 #ifdef CONFIG_RFS_ACCEL
2714         err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2715                                     set_feature_arfs);
2716 #endif
2717
2718         return err ? -EINVAL : 0;
2719 }
2720
2721 #define MXL5_HW_MIN_MTU 64
2722 #define MXL5E_MIN_MTU (MXL5_HW_MIN_MTU + ETH_FCS_LEN)
2723
2724 static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2725 {
2726         struct mlx5e_priv *priv = netdev_priv(netdev);
2727         struct mlx5_core_dev *mdev = priv->mdev;
2728         bool was_opened;
2729         u16 max_mtu;
2730         u16 min_mtu;
2731         int err = 0;
2732         bool reset;
2733
2734         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2735
2736         max_mtu = MLX5E_HW2SW_MTU(max_mtu);
2737         min_mtu = MLX5E_HW2SW_MTU(MXL5E_MIN_MTU);
2738
2739         if (new_mtu > max_mtu || new_mtu < min_mtu) {
2740                 netdev_err(netdev,
2741                            "%s: Bad MTU (%d), valid range is: [%d..%d]\n",
2742                            __func__, new_mtu, min_mtu, max_mtu);
2743                 return -EINVAL;
2744         }
2745
2746         mutex_lock(&priv->state_lock);
2747
2748         reset = !priv->params.lro_en &&
2749                 (priv->params.rq_wq_type !=
2750                  MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
2751
2752         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2753         if (was_opened && reset)
2754                 mlx5e_close_locked(netdev);
2755
2756         netdev->mtu = new_mtu;
2757         mlx5e_set_dev_port_mtu(netdev);
2758
2759         if (was_opened && reset)
2760                 err = mlx5e_open_locked(netdev);
2761
2762         mutex_unlock(&priv->state_lock);
2763
2764         return err;
2765 }
2766
2767 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2768 {
2769         switch (cmd) {
2770         case SIOCSHWTSTAMP:
2771                 return mlx5e_hwstamp_set(dev, ifr);
2772         case SIOCGHWTSTAMP:
2773                 return mlx5e_hwstamp_get(dev, ifr);
2774         default:
2775                 return -EOPNOTSUPP;
2776         }
2777 }
2778
2779 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2780 {
2781         struct mlx5e_priv *priv = netdev_priv(dev);
2782         struct mlx5_core_dev *mdev = priv->mdev;
2783
2784         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2785 }
2786
2787 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2788 {
2789         struct mlx5e_priv *priv = netdev_priv(dev);
2790         struct mlx5_core_dev *mdev = priv->mdev;
2791
2792         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2793                                            vlan, qos);
2794 }
2795
2796 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2797 {
2798         struct mlx5e_priv *priv = netdev_priv(dev);
2799         struct mlx5_core_dev *mdev = priv->mdev;
2800
2801         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2802 }
2803
2804 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2805 {
2806         struct mlx5e_priv *priv = netdev_priv(dev);
2807         struct mlx5_core_dev *mdev = priv->mdev;
2808
2809         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2810 }
2811 static int mlx5_vport_link2ifla(u8 esw_link)
2812 {
2813         switch (esw_link) {
2814         case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
2815                 return IFLA_VF_LINK_STATE_DISABLE;
2816         case MLX5_ESW_VPORT_ADMIN_STATE_UP:
2817                 return IFLA_VF_LINK_STATE_ENABLE;
2818         }
2819         return IFLA_VF_LINK_STATE_AUTO;
2820 }
2821
2822 static int mlx5_ifla_link2vport(u8 ifla_link)
2823 {
2824         switch (ifla_link) {
2825         case IFLA_VF_LINK_STATE_DISABLE:
2826                 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
2827         case IFLA_VF_LINK_STATE_ENABLE:
2828                 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
2829         }
2830         return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
2831 }
2832
2833 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
2834                                    int link_state)
2835 {
2836         struct mlx5e_priv *priv = netdev_priv(dev);
2837         struct mlx5_core_dev *mdev = priv->mdev;
2838
2839         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
2840                                             mlx5_ifla_link2vport(link_state));
2841 }
2842
2843 static int mlx5e_get_vf_config(struct net_device *dev,
2844                                int vf, struct ifla_vf_info *ivi)
2845 {
2846         struct mlx5e_priv *priv = netdev_priv(dev);
2847         struct mlx5_core_dev *mdev = priv->mdev;
2848         int err;
2849
2850         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
2851         if (err)
2852                 return err;
2853         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
2854         return 0;
2855 }
2856
2857 static int mlx5e_get_vf_stats(struct net_device *dev,
2858                               int vf, struct ifla_vf_stats *vf_stats)
2859 {
2860         struct mlx5e_priv *priv = netdev_priv(dev);
2861         struct mlx5_core_dev *mdev = priv->mdev;
2862
2863         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
2864                                             vf_stats);
2865 }
2866
2867 static void mlx5e_add_vxlan_port(struct net_device *netdev,
2868                                  struct udp_tunnel_info *ti)
2869 {
2870         struct mlx5e_priv *priv = netdev_priv(netdev);
2871
2872         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2873                 return;
2874
2875         if (!mlx5e_vxlan_allowed(priv->mdev))
2876                 return;
2877
2878         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
2879 }
2880
2881 static void mlx5e_del_vxlan_port(struct net_device *netdev,
2882                                  struct udp_tunnel_info *ti)
2883 {
2884         struct mlx5e_priv *priv = netdev_priv(netdev);
2885
2886         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2887                 return;
2888
2889         if (!mlx5e_vxlan_allowed(priv->mdev))
2890                 return;
2891
2892         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
2893 }
2894
2895 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
2896                                                     struct sk_buff *skb,
2897                                                     netdev_features_t features)
2898 {
2899         struct udphdr *udph;
2900         u16 proto;
2901         u16 port = 0;
2902
2903         switch (vlan_get_protocol(skb)) {
2904         case htons(ETH_P_IP):
2905                 proto = ip_hdr(skb)->protocol;
2906                 break;
2907         case htons(ETH_P_IPV6):
2908                 proto = ipv6_hdr(skb)->nexthdr;
2909                 break;
2910         default:
2911                 goto out;
2912         }
2913
2914         if (proto == IPPROTO_UDP) {
2915                 udph = udp_hdr(skb);
2916                 port = be16_to_cpu(udph->dest);
2917         }
2918
2919         /* Verify if UDP port is being offloaded by HW */
2920         if (port && mlx5e_vxlan_lookup_port(priv, port))
2921                 return features;
2922
2923 out:
2924         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
2925         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2926 }
2927
2928 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
2929                                               struct net_device *netdev,
2930                                               netdev_features_t features)
2931 {
2932         struct mlx5e_priv *priv = netdev_priv(netdev);
2933
2934         features = vlan_features_check(skb, features);
2935         features = vxlan_features_check(skb, features);
2936
2937         /* Validate if the tunneled packet is being offloaded by HW */
2938         if (skb->encapsulation &&
2939             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
2940                 return mlx5e_vxlan_features_check(priv, skb, features);
2941
2942         return features;
2943 }
2944
2945 static void mlx5e_tx_timeout(struct net_device *dev)
2946 {
2947         struct mlx5e_priv *priv = netdev_priv(dev);
2948         bool sched_work = false;
2949         int i;
2950
2951         netdev_err(dev, "TX timeout detected\n");
2952
2953         for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
2954                 struct mlx5e_sq *sq = priv->txq_to_sq_map[i];
2955
2956                 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
2957                         continue;
2958                 sched_work = true;
2959                 set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
2960                 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
2961                            i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
2962         }
2963
2964         if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
2965                 schedule_work(&priv->tx_timeout_work);
2966 }
2967
2968 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
2969 {
2970         struct mlx5e_priv *priv = netdev_priv(netdev);
2971         struct bpf_prog *old_prog;
2972         int err = 0;
2973         bool reset, was_opened;
2974         int i;
2975
2976         mutex_lock(&priv->state_lock);
2977
2978         if ((netdev->features & NETIF_F_LRO) && prog) {
2979                 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
2980                 err = -EINVAL;
2981                 goto unlock;
2982         }
2983
2984         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2985         /* no need for full reset when exchanging programs */
2986         reset = (!priv->xdp_prog || !prog);
2987
2988         if (was_opened && reset)
2989                 mlx5e_close_locked(netdev);
2990
2991         /* exchange programs */
2992         old_prog = xchg(&priv->xdp_prog, prog);
2993         if (prog)
2994                 bpf_prog_add(prog, 1);
2995         if (old_prog)
2996                 bpf_prog_put(old_prog);
2997
2998         if (reset) /* change RQ type according to priv->xdp_prog */
2999                 mlx5e_set_rq_priv_params(priv);
3000
3001         if (was_opened && reset)
3002                 mlx5e_open_locked(netdev);
3003
3004         if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
3005                 goto unlock;
3006
3007         /* exchanging programs w/o reset, we update ref counts on behalf
3008          * of the channels RQs here.
3009          */
3010         bpf_prog_add(prog, priv->params.num_channels);
3011         for (i = 0; i < priv->params.num_channels; i++) {
3012                 struct mlx5e_channel *c = priv->channel[i];
3013
3014                 set_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
3015                 napi_synchronize(&c->napi);
3016                 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
3017
3018                 old_prog = xchg(&c->rq.xdp_prog, prog);
3019
3020                 clear_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
3021                 /* napi_schedule in case we have missed anything */
3022                 set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
3023                 napi_schedule(&c->napi);
3024
3025                 if (old_prog)
3026                         bpf_prog_put(old_prog);
3027         }
3028
3029 unlock:
3030         mutex_unlock(&priv->state_lock);
3031         return err;
3032 }
3033
3034 static bool mlx5e_xdp_attached(struct net_device *dev)
3035 {
3036         struct mlx5e_priv *priv = netdev_priv(dev);
3037
3038         return !!priv->xdp_prog;
3039 }
3040
3041 static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
3042 {
3043         switch (xdp->command) {
3044         case XDP_SETUP_PROG:
3045                 return mlx5e_xdp_set(dev, xdp->prog);
3046         case XDP_QUERY_PROG:
3047                 xdp->prog_attached = mlx5e_xdp_attached(dev);
3048                 return 0;
3049         default:
3050                 return -EINVAL;
3051         }
3052 }
3053
3054 static const struct net_device_ops mlx5e_netdev_ops_basic = {
3055         .ndo_open                = mlx5e_open,
3056         .ndo_stop                = mlx5e_close,
3057         .ndo_start_xmit          = mlx5e_xmit,
3058         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3059         .ndo_select_queue        = mlx5e_select_queue,
3060         .ndo_get_stats64         = mlx5e_get_stats,
3061         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3062         .ndo_set_mac_address     = mlx5e_set_mac,
3063         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3064         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3065         .ndo_set_features        = mlx5e_set_features,
3066         .ndo_change_mtu          = mlx5e_change_mtu,
3067         .ndo_do_ioctl            = mlx5e_ioctl,
3068         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3069 #ifdef CONFIG_RFS_ACCEL
3070         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3071 #endif
3072         .ndo_tx_timeout          = mlx5e_tx_timeout,
3073         .ndo_xdp                 = mlx5e_xdp,
3074 };
3075
3076 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
3077         .ndo_open                = mlx5e_open,
3078         .ndo_stop                = mlx5e_close,
3079         .ndo_start_xmit          = mlx5e_xmit,
3080         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3081         .ndo_select_queue        = mlx5e_select_queue,
3082         .ndo_get_stats64         = mlx5e_get_stats,
3083         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3084         .ndo_set_mac_address     = mlx5e_set_mac,
3085         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3086         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3087         .ndo_set_features        = mlx5e_set_features,
3088         .ndo_change_mtu          = mlx5e_change_mtu,
3089         .ndo_do_ioctl            = mlx5e_ioctl,
3090         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
3091         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
3092         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3093         .ndo_features_check      = mlx5e_features_check,
3094 #ifdef CONFIG_RFS_ACCEL
3095         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3096 #endif
3097         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
3098         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
3099         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
3100         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
3101         .ndo_get_vf_config       = mlx5e_get_vf_config,
3102         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
3103         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
3104         .ndo_tx_timeout          = mlx5e_tx_timeout,
3105         .ndo_xdp                 = mlx5e_xdp,
3106 };
3107
3108 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3109 {
3110         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3111                 return -ENOTSUPP;
3112         if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
3113             !MLX5_CAP_GEN(mdev, nic_flow_table) ||
3114             !MLX5_CAP_ETH(mdev, csum_cap) ||
3115             !MLX5_CAP_ETH(mdev, max_lso_cap) ||
3116             !MLX5_CAP_ETH(mdev, vlan_cap) ||
3117             !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
3118             MLX5_CAP_FLOWTABLE(mdev,
3119                                flow_table_properties_nic_receive.max_ft_level)
3120                                < 3) {
3121                 mlx5_core_warn(mdev,
3122                                "Not creating net device, some required device capabilities are missing\n");
3123                 return -ENOTSUPP;
3124         }
3125         if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
3126                 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
3127         if (!MLX5_CAP_GEN(mdev, cq_moderation))
3128                 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
3129
3130         return 0;
3131 }
3132
3133 u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3134 {
3135         int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
3136
3137         return bf_buf_size -
3138                sizeof(struct mlx5e_tx_wqe) +
3139                2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3140 }
3141
3142 #ifdef CONFIG_MLX5_CORE_EN_DCB
3143 static void mlx5e_ets_init(struct mlx5e_priv *priv)
3144 {
3145         int i;
3146
3147         priv->params.ets.ets_cap = mlx5_max_tc(priv->mdev) + 1;
3148         for (i = 0; i < priv->params.ets.ets_cap; i++) {
3149                 priv->params.ets.tc_tx_bw[i] = MLX5E_MAX_BW_ALLOC;
3150                 priv->params.ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR;
3151                 priv->params.ets.prio_tc[i] = i;
3152         }
3153
3154         /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
3155         priv->params.ets.prio_tc[0] = 1;
3156         priv->params.ets.prio_tc[1] = 0;
3157 }
3158 #endif
3159
3160 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3161                                    u32 *indirection_rqt, int len,
3162                                    int num_channels)
3163 {
3164         int node = mdev->priv.numa_node;
3165         int node_num_of_cores;
3166         int i;
3167
3168         if (node == -1)
3169                 node = first_online_node;
3170
3171         node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3172
3173         if (node_num_of_cores)
3174                 num_channels = min_t(int, num_channels, node_num_of_cores);
3175
3176         for (i = 0; i < len; i++)
3177                 indirection_rqt[i] = i % num_channels;
3178 }
3179
3180 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3181 {
3182         enum pcie_link_width width;
3183         enum pci_bus_speed speed;
3184         int err = 0;
3185
3186         err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3187         if (err)
3188                 return err;
3189
3190         if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3191                 return -EINVAL;
3192
3193         switch (speed) {
3194         case PCIE_SPEED_2_5GT:
3195                 *pci_bw = 2500 * width;
3196                 break;
3197         case PCIE_SPEED_5_0GT:
3198                 *pci_bw = 5000 * width;
3199                 break;
3200         case PCIE_SPEED_8_0GT:
3201                 *pci_bw = 8000 * width;
3202                 break;
3203         default:
3204                 return -EINVAL;
3205         }
3206
3207         return 0;
3208 }
3209
3210 static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3211 {
3212         return (link_speed && pci_bw &&
3213                 (pci_bw < 40000) && (pci_bw < link_speed));
3214 }
3215
3216 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3217 {
3218         params->rx_cq_period_mode = cq_period_mode;
3219
3220         params->rx_cq_moderation.pkts =
3221                 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3222         params->rx_cq_moderation.usec =
3223                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3224
3225         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3226                 params->rx_cq_moderation.usec =
3227                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
3228 }
3229
3230 static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev,
3231                                    u8 *min_inline_mode)
3232 {
3233         switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
3234         case MLX5E_INLINE_MODE_L2:
3235                 *min_inline_mode = MLX5_INLINE_MODE_L2;
3236                 break;
3237         case MLX5E_INLINE_MODE_VPORT_CONTEXT:
3238                 mlx5_query_nic_vport_min_inline(mdev,
3239                                                 min_inline_mode);
3240                 break;
3241         case MLX5_INLINE_MODE_NOT_REQUIRED:
3242                 *min_inline_mode = MLX5_INLINE_MODE_NONE;
3243                 break;
3244         }
3245 }
3246
3247 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3248                                         struct net_device *netdev,
3249                                         const struct mlx5e_profile *profile,
3250                                         void *ppriv)
3251 {
3252         struct mlx5e_priv *priv = netdev_priv(netdev);
3253         u32 link_speed = 0;
3254         u32 pci_bw = 0;
3255         u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3256                                          MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3257                                          MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
3258
3259         priv->mdev                         = mdev;
3260         priv->netdev                       = netdev;
3261         priv->params.num_channels          = profile->max_nch(mdev);
3262         priv->profile                      = profile;
3263         priv->ppriv                        = ppriv;
3264
3265         priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3266
3267         /* set CQE compression */
3268         priv->params.rx_cqe_compress_admin = false;
3269         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
3270             MLX5_CAP_GEN(mdev, vport_group_manager)) {
3271                 mlx5e_get_max_linkspeed(mdev, &link_speed);
3272                 mlx5e_get_pci_bw(mdev, &pci_bw);
3273                 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3274                               link_speed, pci_bw);
3275                 priv->params.rx_cqe_compress_admin =
3276                         cqe_compress_heuristic(link_speed, pci_bw);
3277         }
3278         priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
3279
3280         mlx5e_set_rq_priv_params(priv);
3281         if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
3282                 priv->params.lro_en = true;
3283
3284         priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3285         mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
3286
3287         priv->params.tx_cq_moderation.usec =
3288                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3289         priv->params.tx_cq_moderation.pkts =
3290                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3291         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
3292         mlx5e_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3293         priv->params.num_tc                = 1;
3294         priv->params.rss_hfunc             = ETH_RSS_HASH_XOR;
3295
3296         netdev_rss_key_fill(priv->params.toeplitz_hash_key,
3297                             sizeof(priv->params.toeplitz_hash_key));
3298
3299         mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
3300                                       MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
3301
3302         priv->params.lro_wqe_sz =
3303                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ -
3304                 /* Extra room needed for build_skb */
3305                 MLX5_RX_HEADROOM -
3306                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3307
3308         /* Initialize pflags */
3309         MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3310                             priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
3311
3312 #ifdef CONFIG_MLX5_CORE_EN_DCB
3313         mlx5e_ets_init(priv);
3314 #endif
3315
3316         mutex_init(&priv->state_lock);
3317
3318         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3319         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3320         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
3321         INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3322 }
3323
3324 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3325 {
3326         struct mlx5e_priv *priv = netdev_priv(netdev);
3327
3328         mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
3329         if (is_zero_ether_addr(netdev->dev_addr) &&
3330             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3331                 eth_hw_addr_random(netdev);
3332                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3333         }
3334 }
3335
3336 static const struct switchdev_ops mlx5e_switchdev_ops = {
3337         .switchdev_port_attr_get        = mlx5e_attr_get,
3338 };
3339
3340 static void mlx5e_build_nic_netdev(struct net_device *netdev)
3341 {
3342         struct mlx5e_priv *priv = netdev_priv(netdev);
3343         struct mlx5_core_dev *mdev = priv->mdev;
3344         bool fcs_supported;
3345         bool fcs_enabled;
3346
3347         SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
3348
3349         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3350                 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
3351 #ifdef CONFIG_MLX5_CORE_EN_DCB
3352                 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
3353 #endif
3354         } else {
3355                 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
3356         }
3357
3358         netdev->watchdog_timeo    = 15 * HZ;
3359
3360         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
3361
3362         netdev->vlan_features    |= NETIF_F_SG;
3363         netdev->vlan_features    |= NETIF_F_IP_CSUM;
3364         netdev->vlan_features    |= NETIF_F_IPV6_CSUM;
3365         netdev->vlan_features    |= NETIF_F_GRO;
3366         netdev->vlan_features    |= NETIF_F_TSO;
3367         netdev->vlan_features    |= NETIF_F_TSO6;
3368         netdev->vlan_features    |= NETIF_F_RXCSUM;
3369         netdev->vlan_features    |= NETIF_F_RXHASH;
3370
3371         if (!!MLX5_CAP_ETH(mdev, lro_cap))
3372                 netdev->vlan_features    |= NETIF_F_LRO;
3373
3374         netdev->hw_features       = netdev->vlan_features;
3375         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
3376         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
3377         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
3378
3379         if (mlx5e_vxlan_allowed(mdev)) {
3380                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
3381                                            NETIF_F_GSO_UDP_TUNNEL_CSUM |
3382                                            NETIF_F_GSO_PARTIAL;
3383                 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
3384                 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
3385                 netdev->hw_enc_features |= NETIF_F_TSO;
3386                 netdev->hw_enc_features |= NETIF_F_TSO6;
3387                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
3388                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3389                                            NETIF_F_GSO_PARTIAL;
3390                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
3391         }
3392
3393         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
3394
3395         if (fcs_supported)
3396                 netdev->hw_features |= NETIF_F_RXALL;
3397
3398         netdev->features          = netdev->hw_features;
3399         if (!priv->params.lro_en)
3400                 netdev->features  &= ~NETIF_F_LRO;
3401
3402         if (fcs_enabled)
3403                 netdev->features  &= ~NETIF_F_RXALL;
3404
3405 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3406         if (FT_CAP(flow_modify_en) &&
3407             FT_CAP(modify_root) &&
3408             FT_CAP(identified_miss_table_mode) &&
3409             FT_CAP(flow_table_modify)) {
3410                 netdev->hw_features      |= NETIF_F_HW_TC;
3411 #ifdef CONFIG_RFS_ACCEL
3412                 netdev->hw_features      |= NETIF_F_NTUPLE;
3413 #endif
3414         }
3415
3416         netdev->features         |= NETIF_F_HIGHDMA;
3417
3418         netdev->priv_flags       |= IFF_UNICAST_FLT;
3419
3420         mlx5e_set_netdev_dev_addr(netdev);
3421
3422 #ifdef CONFIG_NET_SWITCHDEV
3423         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3424                 netdev->switchdev_ops = &mlx5e_switchdev_ops;
3425 #endif
3426 }
3427
3428 static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
3429 {
3430         struct mlx5_core_dev *mdev = priv->mdev;
3431         int err;
3432
3433         err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
3434         if (err) {
3435                 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
3436                 priv->q_counter = 0;
3437         }
3438 }
3439
3440 static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
3441 {
3442         if (!priv->q_counter)
3443                 return;
3444
3445         mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
3446 }
3447
3448 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
3449 {
3450         struct mlx5_core_dev *mdev = priv->mdev;
3451         u64 npages = MLX5E_REQUIRED_MTTS(priv->profile->max_nch(mdev),
3452                                          BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW));
3453         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3454         void *mkc;
3455         u32 *in;
3456         int err;
3457
3458         in = mlx5_vzalloc(inlen);
3459         if (!in)
3460                 return -ENOMEM;
3461
3462         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3463
3464         npages = min_t(u32, ALIGN(U16_MAX, 4) * 2, npages);
3465
3466         MLX5_SET(mkc, mkc, free, 1);
3467         MLX5_SET(mkc, mkc, umr_en, 1);
3468         MLX5_SET(mkc, mkc, lw, 1);
3469         MLX5_SET(mkc, mkc, lr, 1);
3470         MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
3471
3472         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3473         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
3474         MLX5_SET64(mkc, mkc, len, npages << PAGE_SHIFT);
3475         MLX5_SET(mkc, mkc, translations_octword_size,
3476                  MLX5_MTT_OCTW(npages));
3477         MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
3478
3479         err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen);
3480
3481         kvfree(in);
3482         return err;
3483 }
3484
3485 static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
3486                            struct net_device *netdev,
3487                            const struct mlx5e_profile *profile,
3488                            void *ppriv)
3489 {
3490         struct mlx5e_priv *priv = netdev_priv(netdev);
3491
3492         mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
3493         mlx5e_build_nic_netdev(netdev);
3494         mlx5e_vxlan_init(priv);
3495 }
3496
3497 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
3498 {
3499         struct mlx5_core_dev *mdev = priv->mdev;
3500         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3501
3502         mlx5e_vxlan_cleanup(priv);
3503
3504         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3505                 mlx5_eswitch_unregister_vport_rep(esw, 0);
3506 }
3507
3508 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
3509 {
3510         struct mlx5_core_dev *mdev = priv->mdev;
3511         int err;
3512         int i;
3513
3514         err = mlx5e_create_indirect_rqts(priv);
3515         if (err) {
3516                 mlx5_core_warn(mdev, "create indirect rqts failed, %d\n", err);
3517                 return err;
3518         }
3519
3520         err = mlx5e_create_direct_rqts(priv);
3521         if (err) {
3522                 mlx5_core_warn(mdev, "create direct rqts failed, %d\n", err);
3523                 goto err_destroy_indirect_rqts;
3524         }
3525
3526         err = mlx5e_create_indirect_tirs(priv);
3527         if (err) {
3528                 mlx5_core_warn(mdev, "create indirect tirs failed, %d\n", err);
3529                 goto err_destroy_direct_rqts;
3530         }
3531
3532         err = mlx5e_create_direct_tirs(priv);
3533         if (err) {
3534                 mlx5_core_warn(mdev, "create direct tirs failed, %d\n", err);
3535                 goto err_destroy_indirect_tirs;
3536         }
3537
3538         err = mlx5e_create_flow_steering(priv);
3539         if (err) {
3540                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
3541                 goto err_destroy_direct_tirs;
3542         }
3543
3544         err = mlx5e_tc_init(priv);
3545         if (err)
3546                 goto err_destroy_flow_steering;
3547
3548         return 0;
3549
3550 err_destroy_flow_steering:
3551         mlx5e_destroy_flow_steering(priv);
3552 err_destroy_direct_tirs:
3553         mlx5e_destroy_direct_tirs(priv);
3554 err_destroy_indirect_tirs:
3555         mlx5e_destroy_indirect_tirs(priv);
3556 err_destroy_direct_rqts:
3557         for (i = 0; i < priv->profile->max_nch(mdev); i++)
3558                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3559 err_destroy_indirect_rqts:
3560         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3561         return err;
3562 }
3563
3564 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
3565 {
3566         int i;
3567
3568         mlx5e_tc_cleanup(priv);
3569         mlx5e_destroy_flow_steering(priv);
3570         mlx5e_destroy_direct_tirs(priv);
3571         mlx5e_destroy_indirect_tirs(priv);
3572         for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
3573                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3574         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3575 }
3576
3577 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
3578 {
3579         int err;
3580
3581         err = mlx5e_create_tises(priv);
3582         if (err) {
3583                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
3584                 return err;
3585         }
3586
3587 #ifdef CONFIG_MLX5_CORE_EN_DCB
3588         mlx5e_dcbnl_ieee_setets_core(priv, &priv->params.ets);
3589 #endif
3590         return 0;
3591 }
3592
3593 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
3594 {
3595         struct net_device *netdev = priv->netdev;
3596         struct mlx5_core_dev *mdev = priv->mdev;
3597         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3598         struct mlx5_eswitch_rep rep;
3599
3600         mlx5_lag_add(mdev, netdev);
3601
3602         if (mlx5e_vxlan_allowed(mdev)) {
3603                 rtnl_lock();
3604                 udp_tunnel_get_rx_info(netdev);
3605                 rtnl_unlock();
3606         }
3607
3608         mlx5e_enable_async_events(priv);
3609         queue_work(priv->wq, &priv->set_rx_mode_work);
3610
3611         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3612                 mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
3613                 rep.load = mlx5e_nic_rep_load;
3614                 rep.unload = mlx5e_nic_rep_unload;
3615                 rep.vport = 0;
3616                 rep.priv_data = priv;
3617                 mlx5_eswitch_register_vport_rep(esw, &rep);
3618         }
3619 }
3620
3621 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
3622 {
3623         queue_work(priv->wq, &priv->set_rx_mode_work);
3624         mlx5e_disable_async_events(priv);
3625         mlx5_lag_remove(priv->mdev);
3626 }
3627
3628 static const struct mlx5e_profile mlx5e_nic_profile = {
3629         .init              = mlx5e_nic_init,
3630         .cleanup           = mlx5e_nic_cleanup,
3631         .init_rx           = mlx5e_init_nic_rx,
3632         .cleanup_rx        = mlx5e_cleanup_nic_rx,
3633         .init_tx           = mlx5e_init_nic_tx,
3634         .cleanup_tx        = mlx5e_cleanup_nic_tx,
3635         .enable            = mlx5e_nic_enable,
3636         .disable           = mlx5e_nic_disable,
3637         .update_stats      = mlx5e_update_stats,
3638         .max_nch           = mlx5e_get_max_num_channels,
3639         .max_tc            = MLX5E_MAX_NUM_TC,
3640 };
3641
3642 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
3643                                        const struct mlx5e_profile *profile,
3644                                        void *ppriv)
3645 {
3646         int nch = profile->max_nch(mdev);
3647         struct net_device *netdev;
3648         struct mlx5e_priv *priv;
3649
3650         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
3651                                     nch * profile->max_tc,
3652                                     nch);
3653         if (!netdev) {
3654                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
3655                 return NULL;
3656         }
3657
3658         profile->init(mdev, netdev, profile, ppriv);
3659
3660         netif_carrier_off(netdev);
3661
3662         priv = netdev_priv(netdev);
3663
3664         priv->wq = create_singlethread_workqueue("mlx5e");
3665         if (!priv->wq)
3666                 goto err_cleanup_nic;
3667
3668         return netdev;
3669
3670 err_cleanup_nic:
3671         profile->cleanup(priv);
3672         free_netdev(netdev);
3673
3674         return NULL;
3675 }
3676
3677 int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3678 {
3679         const struct mlx5e_profile *profile;
3680         struct mlx5e_priv *priv;
3681         int err;
3682
3683         priv = netdev_priv(netdev);
3684         profile = priv->profile;
3685         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
3686
3687         err = mlx5e_create_umr_mkey(priv);
3688         if (err) {
3689                 mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
3690                 goto out;
3691         }
3692
3693         err = profile->init_tx(priv);
3694         if (err)
3695                 goto err_destroy_umr_mkey;
3696
3697         err = mlx5e_open_drop_rq(priv);
3698         if (err) {
3699                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
3700                 goto err_cleanup_tx;
3701         }
3702
3703         err = profile->init_rx(priv);
3704         if (err)
3705                 goto err_close_drop_rq;
3706
3707         mlx5e_create_q_counter(priv);
3708
3709         mlx5e_init_l2_addr(priv);
3710
3711         mlx5e_set_dev_port_mtu(netdev);
3712
3713         if (profile->enable)
3714                 profile->enable(priv);
3715
3716         rtnl_lock();
3717         if (netif_running(netdev))
3718                 mlx5e_open(netdev);
3719         netif_device_attach(netdev);
3720         rtnl_unlock();
3721
3722         return 0;
3723
3724 err_close_drop_rq:
3725         mlx5e_close_drop_rq(priv);
3726
3727 err_cleanup_tx:
3728         profile->cleanup_tx(priv);
3729
3730 err_destroy_umr_mkey:
3731         mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
3732
3733 out:
3734         return err;
3735 }
3736
3737 static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
3738 {
3739         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3740         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3741         int vport;
3742         u8 mac[ETH_ALEN];
3743
3744         if (!MLX5_CAP_GEN(mdev, vport_group_manager))
3745                 return;
3746
3747         mlx5_query_nic_vport_mac_address(mdev, 0, mac);
3748
3749         for (vport = 1; vport < total_vfs; vport++) {
3750                 struct mlx5_eswitch_rep rep;
3751
3752                 rep.load = mlx5e_vport_rep_load;
3753                 rep.unload = mlx5e_vport_rep_unload;
3754                 rep.vport = vport;
3755                 ether_addr_copy(rep.hw_id, mac);
3756                 mlx5_eswitch_register_vport_rep(esw, &rep);
3757         }
3758 }
3759
3760 void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3761 {
3762         struct mlx5e_priv *priv = netdev_priv(netdev);
3763         const struct mlx5e_profile *profile = priv->profile;
3764
3765         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
3766         if (profile->disable)
3767                 profile->disable(priv);
3768
3769         flush_workqueue(priv->wq);
3770
3771         rtnl_lock();
3772         if (netif_running(netdev))
3773                 mlx5e_close(netdev);
3774         netif_device_detach(netdev);
3775         rtnl_unlock();
3776
3777         mlx5e_destroy_q_counter(priv);
3778         profile->cleanup_rx(priv);
3779         mlx5e_close_drop_rq(priv);
3780         profile->cleanup_tx(priv);
3781         mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
3782         cancel_delayed_work_sync(&priv->update_stats_work);
3783 }
3784
3785 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
3786  * hardware contexts and to connect it to the current netdev.
3787  */
3788 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
3789 {
3790         struct mlx5e_priv *priv = vpriv;
3791         struct net_device *netdev = priv->netdev;
3792         int err;
3793
3794         if (netif_device_present(netdev))
3795                 return 0;
3796
3797         err = mlx5e_create_mdev_resources(mdev);
3798         if (err)
3799                 return err;
3800
3801         err = mlx5e_attach_netdev(mdev, netdev);
3802         if (err) {
3803                 mlx5e_destroy_mdev_resources(mdev);
3804                 return err;
3805         }
3806
3807         return 0;
3808 }
3809
3810 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
3811 {
3812         struct mlx5e_priv *priv = vpriv;
3813         struct net_device *netdev = priv->netdev;
3814
3815         if (!netif_device_present(netdev))
3816                 return;
3817
3818         mlx5e_detach_netdev(mdev, netdev);
3819         mlx5e_destroy_mdev_resources(mdev);
3820 }
3821
3822 static void *mlx5e_add(struct mlx5_core_dev *mdev)
3823 {
3824         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3825         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3826         void *ppriv = NULL;
3827         void *priv;
3828         int vport;
3829         int err;
3830         struct net_device *netdev;
3831
3832         err = mlx5e_check_required_hca_cap(mdev);
3833         if (err)
3834                 return NULL;
3835
3836         mlx5e_register_vport_rep(mdev);
3837
3838         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3839                 ppriv = &esw->offloads.vport_reps[0];
3840
3841         netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
3842         if (!netdev) {
3843                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
3844                 goto err_unregister_reps;
3845         }
3846
3847         priv = netdev_priv(netdev);
3848
3849         err = mlx5e_attach(mdev, priv);
3850         if (err) {
3851                 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
3852                 goto err_destroy_netdev;
3853         }
3854
3855         err = register_netdev(netdev);
3856         if (err) {
3857                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
3858                 goto err_detach;
3859         }
3860
3861         return priv;
3862
3863 err_detach:
3864         mlx5e_detach(mdev, priv);
3865
3866 err_destroy_netdev:
3867         mlx5e_destroy_netdev(mdev, priv);
3868
3869 err_unregister_reps:
3870         for (vport = 1; vport < total_vfs; vport++)
3871                 mlx5_eswitch_unregister_vport_rep(esw, vport);
3872
3873         return NULL;
3874 }
3875
3876 void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
3877 {
3878         const struct mlx5e_profile *profile = priv->profile;
3879         struct net_device *netdev = priv->netdev;
3880
3881         unregister_netdev(netdev);
3882         destroy_workqueue(priv->wq);
3883         if (profile->cleanup)
3884                 profile->cleanup(priv);
3885         free_netdev(netdev);
3886 }
3887
3888 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
3889 {
3890         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3891         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3892         struct mlx5e_priv *priv = vpriv;
3893         int vport;
3894
3895         for (vport = 1; vport < total_vfs; vport++)
3896                 mlx5_eswitch_unregister_vport_rep(esw, vport);
3897
3898         mlx5e_detach(mdev, vpriv);
3899         mlx5e_destroy_netdev(mdev, priv);
3900 }
3901
3902 static void *mlx5e_get_netdev(void *vpriv)
3903 {
3904         struct mlx5e_priv *priv = vpriv;
3905
3906         return priv->netdev;
3907 }
3908
3909 static struct mlx5_interface mlx5e_interface = {
3910         .add       = mlx5e_add,
3911         .remove    = mlx5e_remove,
3912         .attach    = mlx5e_attach,
3913         .detach    = mlx5e_detach,
3914         .event     = mlx5e_async_event,
3915         .protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
3916         .get_dev   = mlx5e_get_netdev,
3917 };
3918
3919 void mlx5e_init(void)
3920 {
3921         mlx5e_build_ptys2ethtool_map();
3922         mlx5_register_interface(&mlx5e_interface);
3923 }
3924
3925 void mlx5e_cleanup(void)
3926 {
3927         mlx5_unregister_interface(&mlx5e_interface);
3928 }