drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include "en.h"
  38 #include "en_tc.h"
  39 #include "eswitch.h"
  40 #include "vxlan.h"
  41
  42 struct mlx5e_rq_param {
  43         u32                     rqc[MLX5_ST_SZ_DW(rqc)];
  44         struct mlx5_wq_param    wq;
  45         bool                    am_enabled;
  46 };
  47
  48 struct mlx5e_sq_param {
  49         u32                        sqc[MLX5_ST_SZ_DW(sqc)];
  50         struct mlx5_wq_param       wq;
  51         u16                        max_inline;
  52         u8                         min_inline_mode;
  53         bool                       icosq;
  54 };
  55
  56 struct mlx5e_cq_param {
  57         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
  58         struct mlx5_wq_param       wq;
  59         u16                        eq_ix;
  60         u8                         cq_period_mode;
  61 };
  62
  63 struct mlx5e_channel_param {
  64         struct mlx5e_rq_param      rq;
  65         struct mlx5e_sq_param      sq;
  66         struct mlx5e_sq_param      icosq;
  67         struct mlx5e_cq_param      rx_cq;
  68         struct mlx5e_cq_param      tx_cq;
  69         struct mlx5e_cq_param      icosq_cq;
  70 };
  71
  72 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  73 {
  74         return MLX5_CAP_GEN(mdev, striding_rq) &&
  75                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  76                 MLX5_CAP_ETH(mdev, reg_umr_sq);
  77 }
  78
  79 static void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
  80 {
  81         priv->params.rq_wq_type = rq_type;
  82         switch (priv->params.rq_wq_type) {
  83         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
  84                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
  85                 priv->params.mpwqe_log_stride_sz = priv->params.rx_cqe_compress ?
  86                         MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
  87                         MLX5_MPWRQ_LOG_STRIDE_SIZE;
  88                 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
  89                         priv->params.mpwqe_log_stride_sz;
  90                 break;
  91         default: /* MLX5_WQ_TYPE_LINKED_LIST */
  92                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
  93         }
  94         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
  95                                                BIT(priv->params.log_rq_size));
  96
  97         mlx5_core_info(priv->mdev,
  98                        "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
  99                        priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
 100                        BIT(priv->params.log_rq_size),
 101                        BIT(priv->params.mpwqe_log_stride_sz),
 102                        priv->params.rx_cqe_compress_admin);
 103 }
 104
 105 static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
 106 {
 107         u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) ?
 108                     MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 109                     MLX5_WQ_TYPE_LINKED_LIST;
 110         mlx5e_set_rq_type_params(priv, rq_type);
 111 }
 112
 113 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
 114 {
 115         struct mlx5_core_dev *mdev = priv->mdev;
 116         u8 port_state;
 117
 118         port_state = mlx5_query_vport_state(mdev,
 119                 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 120
 121         if (port_state == VPORT_STATE_UP) {
 122                 netdev_info(priv->netdev, "Link up\n");
 123                 netif_carrier_on(priv->netdev);
 124         } else {
 125                 netdev_info(priv->netdev, "Link down\n");
 126                 netif_carrier_off(priv->netdev);
 127         }
 128 }
 129
 130 static void mlx5e_update_carrier_work(struct work_struct *work)
 131 {
 132         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 133                                                update_carrier_work);
 134
 135         mutex_lock(&priv->state_lock);
 136         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 137                 mlx5e_update_carrier(priv);
 138         mutex_unlock(&priv->state_lock);
 139 }
 140
 141 static void mlx5e_tx_timeout_work(struct work_struct *work)
 142 {
 143         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 144                                                tx_timeout_work);
 145         int err;
 146
 147         rtnl_lock();
 148         mutex_lock(&priv->state_lock);
 149         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 150                 goto unlock;
 151         mlx5e_close_locked(priv->netdev);
 152         err = mlx5e_open_locked(priv->netdev);
 153         if (err)
 154                 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
 155                            err);
 156 unlock:
 157         mutex_unlock(&priv->state_lock);
 158         rtnl_unlock();
 159 }
 160
 161 static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 162 {
 163         struct mlx5e_sw_stats *s = &priv->stats.sw;
 164         struct mlx5e_rq_stats *rq_stats;
 165         struct mlx5e_sq_stats *sq_stats;
 166         u64 tx_offload_none = 0;
 167         int i, j;
 168
 169         memset(s, 0, sizeof(*s));
 170         for (i = 0; i < priv->params.num_channels; i++) {
 171                 rq_stats = &priv->channel[i]->rq.stats;
 172
 173                 s->rx_packets   += rq_stats->packets;
 174                 s->rx_bytes     += rq_stats->bytes;
 175                 s->rx_lro_packets += rq_stats->lro_packets;
 176                 s->rx_lro_bytes += rq_stats->lro_bytes;
 177                 s->rx_csum_none += rq_stats->csum_none;
 178                 s->rx_csum_complete += rq_stats->csum_complete;
 179                 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
 180                 s->rx_wqe_err   += rq_stats->wqe_err;
 181                 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
 182                 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
 183                 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
 184                 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
 185                 s->rx_cache_reuse += rq_stats->cache_reuse;
 186                 s->rx_cache_full  += rq_stats->cache_full;
 187                 s->rx_cache_empty += rq_stats->cache_empty;
 188                 s->rx_cache_busy  += rq_stats->cache_busy;
 189
 190                 for (j = 0; j < priv->params.num_tc; j++) {
 191                         sq_stats = &priv->channel[i]->sq[j].stats;
 192
 193                         s->tx_packets           += sq_stats->packets;
 194                         s->tx_bytes             += sq_stats->bytes;
 195                         s->tx_tso_packets       += sq_stats->tso_packets;
 196                         s->tx_tso_bytes         += sq_stats->tso_bytes;
 197                         s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
 198                         s->tx_tso_inner_bytes   += sq_stats->tso_inner_bytes;
 199                         s->tx_queue_stopped     += sq_stats->stopped;
 200                         s->tx_queue_wake        += sq_stats->wake;
 201                         s->tx_queue_dropped     += sq_stats->dropped;
 202                         s->tx_xmit_more         += sq_stats->xmit_more;
 203                         s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
 204                         tx_offload_none         += sq_stats->csum_none;
 205                 }
 206         }
 207
 208         /* Update calculated offload counters */
 209         s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
 210         s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
 211
 212         s->link_down_events_phy = MLX5_GET(ppcnt_reg,
 213                                 priv->stats.pport.phy_counters,
 214                                 counter_set.phys_layer_cntrs.link_down_events);
 215 }
 216
 217 static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
 218 {
 219         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 220         u32 *out = (u32 *)priv->stats.vport.query_vport_out;
 221         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
 222         struct mlx5_core_dev *mdev = priv->mdev;
 223
 224         MLX5_SET(query_vport_counter_in, in, opcode,
 225                  MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 226         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 227         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 228
 229         memset(out, 0, outlen);
 230         mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 231 }
 232
 233 static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 234 {
 235         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
 236         struct mlx5_core_dev *mdev = priv->mdev;
 237         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 238         int prio;
 239         void *out;
 240         u32 *in;
 241
 242         in = mlx5_vzalloc(sz);
 243         if (!in)
 244                 goto free_out;
 245
 246         MLX5_SET(ppcnt_reg, in, local_port, 1);
 247
 248         out = pstats->IEEE_802_3_counters;
 249         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 250         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 251
 252         out = pstats->RFC_2863_counters;
 253         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 254         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 255
 256         out = pstats->RFC_2819_counters;
 257         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 258         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 259
 260         out = pstats->phy_counters;
 261         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 262         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 263
 264         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 265         for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 266                 out = pstats->per_prio_counters[prio];
 267                 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
 268                 mlx5_core_access_reg(mdev, in, sz, out, sz,
 269                                      MLX5_REG_PPCNT, 0, 0);
 270         }
 271
 272 free_out:
 273         kvfree(in);
 274 }
 275
 276 static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
 277 {
 278         struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
 279
 280         if (!priv->q_counter)
 281                 return;
 282
 283         mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
 284                                       &qcnt->rx_out_of_buffer);
 285 }
 286
 287 void mlx5e_update_stats(struct mlx5e_priv *priv)
 288 {
 289         mlx5e_update_q_counter(priv);
 290         mlx5e_update_vport_counters(priv);
 291         mlx5e_update_pport_counters(priv);
 292         mlx5e_update_sw_counters(priv);
 293 }
 294
 295 void mlx5e_update_stats_work(struct work_struct *work)
 296 {
 297         struct delayed_work *dwork = to_delayed_work(work);
 298         struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
 299                                                update_stats_work);
 300         mutex_lock(&priv->state_lock);
 301         if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 302                 priv->profile->update_stats(priv);
 303                 queue_delayed_work(priv->wq, dwork,
 304                                    msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
 305         }
 306         mutex_unlock(&priv->state_lock);
 307 }
 308
 309 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
 310                               enum mlx5_dev_event event, unsigned long param)
 311 {
 312         struct mlx5e_priv *priv = vpriv;
 313
 314         if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
 315                 return;
 316
 317         switch (event) {
 318         case MLX5_DEV_EVENT_PORT_UP:
 319         case MLX5_DEV_EVENT_PORT_DOWN:
 320                 queue_work(priv->wq, &priv->update_carrier_work);
 321                 break;
 322
 323         default:
 324                 break;
 325         }
 326 }
 327
 328 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 329 {
 330         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 331 }
 332
 333 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 334 {
 335         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
 336         synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
 337 }
 338
 339 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 340 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 341
 342 static inline int mlx5e_get_wqe_mtt_sz(void)
 343 {
 344         /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
 345          * To avoid copying garbage after the mtt array, we allocate
 346          * a little more.
 347          */
 348         return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
 349                      MLX5_UMR_MTT_ALIGNMENT);
 350 }
 351
 352 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
 353                                        struct mlx5e_umr_wqe *wqe, u16 ix)
 354 {
 355         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 356         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 357         struct mlx5_wqe_data_seg      *dseg = &wqe->data;
 358         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 359         u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
 360         u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
 361
 362         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 363                                       ds_cnt);
 364         cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
 365         cseg->imm       = rq->mkey_be;
 366
 367         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
 368         ucseg->klm_octowords =
 369                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 370         ucseg->bsf_octowords =
 371                 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
 372         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 373
 374         dseg->lkey = sq->mkey_be;
 375         dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
 376 }
 377
 378 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
 379                                      struct mlx5e_channel *c)
 380 {
 381         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 382         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 383         int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
 384         int i;
 385
 386         rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
 387                                       GFP_KERNEL, cpu_to_node(c->cpu));
 388         if (!rq->mpwqe.info)
 389                 goto err_out;
 390
 391         /* We allocate more than mtt_sz as we will align the pointer */
 392         rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
 393                                         cpu_to_node(c->cpu));
 394         if (unlikely(!rq->mpwqe.mtt_no_align))
 395                 goto err_free_wqe_info;
 396
 397         for (i = 0; i < wq_sz; i++) {
 398                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 399
 400                 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
 401                                         MLX5_UMR_ALIGN);
 402                 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
 403                                                   PCI_DMA_TODEVICE);
 404                 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
 405                         goto err_unmap_mtts;
 406
 407                 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
 408         }
 409
 410         return 0;
 411
 412 err_unmap_mtts:
 413         while (--i >= 0) {
 414                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 415
 416                 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
 417                                  PCI_DMA_TODEVICE);
 418         }
 419         kfree(rq->mpwqe.mtt_no_align);
 420 err_free_wqe_info:
 421         kfree(rq->mpwqe.info);
 422
 423 err_out:
 424         return -ENOMEM;
 425 }
 426
 427 static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
 428 {
 429         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 430         int mtt_sz = mlx5e_get_wqe_mtt_sz();
 431         int i;
 432
 433         for (i = 0; i < wq_sz; i++) {
 434                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
 435
 436                 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
 437                                  PCI_DMA_TODEVICE);
 438         }
 439         kfree(rq->mpwqe.mtt_no_align);
 440         kfree(rq->mpwqe.info);
 441 }
 442
 443 static int mlx5e_create_rq(struct mlx5e_channel *c,
 444                            struct mlx5e_rq_param *param,
 445                            struct mlx5e_rq *rq)
 446 {
 447         struct mlx5e_priv *priv = c->priv;
 448         struct mlx5_core_dev *mdev = priv->mdev;
 449         void *rqc = param->rqc;
 450         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 451         u32 byte_count;
 452         u32 frag_sz;
 453         int npages;
 454         int wq_sz;
 455         int err;
 456         int i;
 457
 458         param->wq.db_numa_node = cpu_to_node(c->cpu);
 459
 460         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
 461                                 &rq->wq_ctrl);
 462         if (err)
 463                 return err;
 464
 465         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
 466
 467         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 468
 469         rq->wq_type = priv->params.rq_wq_type;
 470         rq->pdev    = c->pdev;
 471         rq->netdev  = c->netdev;
 472         rq->tstamp  = &priv->tstamp;
 473         rq->channel = c;
 474         rq->ix      = c->ix;
 475         rq->priv    = c->priv;
 476
 477         switch (priv->params.rq_wq_type) {
 478         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 479                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
 480                 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
 481                 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 482
 483                 rq->mpwqe.mtt_offset = c->ix *
 484                         MLX5E_REQUIRED_MTTS(1, BIT(priv->params.log_rq_size));
 485
 486                 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
 487                 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
 488
 489                 rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
 490                 byte_count = rq->buff.wqe_sz;
 491                 rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
 492                 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
 493                 if (err)
 494                         goto err_rq_wq_destroy;
 495                 break;
 496         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 497                 rq->dma_info = kzalloc_node(wq_sz * sizeof(*rq->dma_info),
 498                                             GFP_KERNEL, cpu_to_node(c->cpu));
 499                 if (!rq->dma_info) {
 500                         err = -ENOMEM;
 501                         goto err_rq_wq_destroy;
 502                 }
 503
 504                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
 505                 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
 506                 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 507
 508                 rq->buff.wqe_sz = (priv->params.lro_en) ?
 509                                 priv->params.lro_wqe_sz :
 510                                 MLX5E_SW2HW_MTU(priv->netdev->mtu);
 511                 byte_count = rq->buff.wqe_sz;
 512
 513                 /* calc the required page order */
 514                 frag_sz = MLX5_RX_HEADROOM +
 515                           byte_count /* packet data */ +
 516                           SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 517                 frag_sz = SKB_DATA_ALIGN(frag_sz);
 518
 519                 npages = DIV_ROUND_UP(frag_sz, PAGE_SIZE);
 520                 rq->buff.page_order = order_base_2(npages);
 521
 522                 byte_count |= MLX5_HW_START_PADDING;
 523                 rq->mkey_be = c->mkey_be;
 524         }
 525
 526         for (i = 0; i < wq_sz; i++) {
 527                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 528
 529                 wqe->data.byte_count = cpu_to_be32(byte_count);
 530                 wqe->data.lkey = rq->mkey_be;
 531         }
 532
 533         INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
 534         rq->am.mode = priv->params.rx_cq_period_mode;
 535
 536         rq->page_cache.head = 0;
 537         rq->page_cache.tail = 0;
 538
 539         return 0;
 540
 541 err_rq_wq_destroy:
 542         mlx5_wq_destroy(&rq->wq_ctrl);
 543
 544         return err;
 545 }
 546
 547 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 548 {
 549         int i;
 550
 551         switch (rq->wq_type) {
 552         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 553                 mlx5e_rq_free_mpwqe_info(rq);
 554                 break;
 555         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 556                 kfree(rq->dma_info);
 557         }
 558
 559         for (i = rq->page_cache.head; i != rq->page_cache.tail;
 560              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 561                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 562
 563                 mlx5e_page_release(rq, dma_info, false);
 564         }
 565         mlx5_wq_destroy(&rq->wq_ctrl);
 566 }
 567
 568 static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 569 {
 570         struct mlx5e_priv *priv = rq->priv;
 571         struct mlx5_core_dev *mdev = priv->mdev;
 572
 573         void *in;
 574         void *rqc;
 575         void *wq;
 576         int inlen;
 577         int err;
 578
 579         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 580                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 581         in = mlx5_vzalloc(inlen);
 582         if (!in)
 583                 return -ENOMEM;
 584
 585         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 586         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 587
 588         memcpy(rqc, param->rqc, sizeof(param->rqc));
 589
 590         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 591         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 592         MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
 593         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 594                                                 MLX5_ADAPTER_PAGE_SHIFT);
 595         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 596
 597         mlx5_fill_page_array(&rq->wq_ctrl.buf,
 598                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 599
 600         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 601
 602         kvfree(in);
 603
 604         return err;
 605 }
 606
 607 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
 608                                  int next_state)
 609 {
 610         struct mlx5e_channel *c = rq->channel;
 611         struct mlx5e_priv *priv = c->priv;
 612         struct mlx5_core_dev *mdev = priv->mdev;
 613
 614         void *in;
 615         void *rqc;
 616         int inlen;
 617         int err;
 618
 619         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 620         in = mlx5_vzalloc(inlen);
 621         if (!in)
 622                 return -ENOMEM;
 623
 624         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 625
 626         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 627         MLX5_SET(rqc, rqc, state, next_state);
 628
 629         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 630
 631         kvfree(in);
 632
 633         return err;
 634 }
 635
 636 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 637 {
 638         struct mlx5e_channel *c = rq->channel;
 639         struct mlx5e_priv *priv = c->priv;
 640         struct mlx5_core_dev *mdev = priv->mdev;
 641
 642         void *in;
 643         void *rqc;
 644         int inlen;
 645         int err;
 646
 647         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 648         in = mlx5_vzalloc(inlen);
 649         if (!in)
 650                 return -ENOMEM;
 651
 652         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 653
 654         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 655         MLX5_SET64(modify_rq_in, in, modify_bitmask,
 656                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 657         MLX5_SET(rqc, rqc, vsd, vsd);
 658         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 659
 660         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 661
 662         kvfree(in);
 663
 664         return err;
 665 }
 666
 667 static void mlx5e_disable_rq(struct mlx5e_rq *rq)
 668 {
 669         mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
 670 }
 671
 672 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
 673 {
 674         unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
 675         struct mlx5e_channel *c = rq->channel;
 676         struct mlx5e_priv *priv = c->priv;
 677         struct mlx5_wq_ll *wq = &rq->wq;
 678
 679         while (time_before(jiffies, exp_time)) {
 680                 if (wq->cur_sz >= priv->params.min_rx_wqes)
 681                         return 0;
 682
 683                 msleep(20);
 684         }
 685
 686         return -ETIMEDOUT;
 687 }
 688
 689 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 690 {
 691         struct mlx5_wq_ll *wq = &rq->wq;
 692         struct mlx5e_rx_wqe *wqe;
 693         __be16 wqe_ix_be;
 694         u16 wqe_ix;
 695
 696         /* UMR WQE (if in progress) is always at wq->head */
 697         if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
 698                 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
 699
 700         while (!mlx5_wq_ll_is_empty(wq)) {
 701                 wqe_ix_be = *wq->tail_next;
 702                 wqe_ix    = be16_to_cpu(wqe_ix_be);
 703                 wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
 704                 rq->dealloc_wqe(rq, wqe_ix);
 705                 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
 706                                &wqe->next.next_wqe_index);
 707         }
 708 }
 709
 710 static int mlx5e_open_rq(struct mlx5e_channel *c,
 711                          struct mlx5e_rq_param *param,
 712                          struct mlx5e_rq *rq)
 713 {
 714         struct mlx5e_sq *sq = &c->icosq;
 715         u16 pi = sq->pc & sq->wq.sz_m1;
 716         int err;
 717
 718         err = mlx5e_create_rq(c, param, rq);
 719         if (err)
 720                 return err;
 721
 722         err = mlx5e_enable_rq(rq, param);
 723         if (err)
 724                 goto err_destroy_rq;
 725
 726         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 727         if (err)
 728                 goto err_disable_rq;
 729
 730         if (param->am_enabled)
 731                 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 732
 733         sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;
 734         sq->ico_wqe_info[pi].num_wqebbs = 1;
 735         mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
 736
 737         return 0;
 738
 739 err_disable_rq:
 740         mlx5e_disable_rq(rq);
 741 err_destroy_rq:
 742         mlx5e_destroy_rq(rq);
 743
 744         return err;
 745 }
 746
 747 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 748 {
 749         set_bit(MLX5E_RQ_STATE_FLUSH, &rq->state);
 750         napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
 751         cancel_work_sync(&rq->am.work);
 752
 753         mlx5e_disable_rq(rq);
 754         mlx5e_free_rx_descs(rq);
 755         mlx5e_destroy_rq(rq);
 756 }
 757
 758 static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
 759 {
 760         kfree(sq->wqe_info);
 761         kfree(sq->dma_fifo);
 762         kfree(sq->skb);
 763 }
 764
 765 static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
 766 {
 767         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 768         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 769
 770         sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);
 771         sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,
 772                                     numa);
 773         sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,
 774                                     numa);
 775
 776         if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {
 777                 mlx5e_free_sq_db(sq);
 778                 return -ENOMEM;
 779         }
 780
 781         sq->dma_fifo_mask = df_sz - 1;
 782
 783         return 0;
 784 }
 785
 786 static int mlx5e_create_sq(struct mlx5e_channel *c,
 787                            int tc,
 788                            struct mlx5e_sq_param *param,
 789                            struct mlx5e_sq *sq)
 790 {
 791         struct mlx5e_priv *priv = c->priv;
 792         struct mlx5_core_dev *mdev = priv->mdev;
 793
 794         void *sqc = param->sqc;
 795         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
 796         int err;
 797
 798         err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));
 799         if (err)
 800                 return err;
 801
 802         param->wq.db_numa_node = cpu_to_node(c->cpu);
 803
 804         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
 805                                  &sq->wq_ctrl);
 806         if (err)
 807                 goto err_unmap_free_uar;
 808
 809         sq->wq.db       = &sq->wq.db[MLX5_SND_DBR];
 810         if (sq->uar.bf_map) {
 811                 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
 812                 sq->uar_map = sq->uar.bf_map;
 813         } else {
 814                 sq->uar_map = sq->uar.map;
 815         }
 816         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
 817         sq->max_inline  = param->max_inline;
 818         sq->min_inline_mode =
 819                 MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5E_INLINE_MODE_VPORT_CONTEXT ?
 820                 param->min_inline_mode : 0;
 821
 822         err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
 823         if (err)
 824                 goto err_sq_wq_destroy;
 825
 826         if (param->icosq) {
 827                 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 828
 829                 sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *
 830                                                 wq_sz,
 831                                                 GFP_KERNEL,
 832                                                 cpu_to_node(c->cpu));
 833                 if (!sq->ico_wqe_info) {
 834                         err = -ENOMEM;
 835                         goto err_free_sq_db;
 836                 }
 837         } else {
 838                 int txq_ix;
 839
 840                 txq_ix = c->ix + tc * priv->params.num_channels;
 841                 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
 842                 priv->txq_to_sq_map[txq_ix] = sq;
 843         }
 844
 845         sq->pdev      = c->pdev;
 846         sq->tstamp    = &priv->tstamp;
 847         sq->mkey_be   = c->mkey_be;
 848         sq->channel   = c;
 849         sq->tc        = tc;
 850         sq->edge      = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
 851         sq->bf_budget = MLX5E_SQ_BF_BUDGET;
 852
 853         return 0;
 854
 855 err_free_sq_db:
 856         mlx5e_free_sq_db(sq);
 857
 858 err_sq_wq_destroy:
 859         mlx5_wq_destroy(&sq->wq_ctrl);
 860
 861 err_unmap_free_uar:
 862         mlx5_unmap_free_uar(mdev, &sq->uar);
 863
 864         return err;
 865 }
 866
 867 static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
 868 {
 869         struct mlx5e_channel *c = sq->channel;
 870         struct mlx5e_priv *priv = c->priv;
 871
 872         kfree(sq->ico_wqe_info);
 873         mlx5e_free_sq_db(sq);
 874         mlx5_wq_destroy(&sq->wq_ctrl);
 875         mlx5_unmap_free_uar(priv->mdev, &sq->uar);
 876 }
 877
 878 static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
 879 {
 880         struct mlx5e_channel *c = sq->channel;
 881         struct mlx5e_priv *priv = c->priv;
 882         struct mlx5_core_dev *mdev = priv->mdev;
 883
 884         void *in;
 885         void *sqc;
 886         void *wq;
 887         int inlen;
 888         int err;
 889
 890         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
 891                 sizeof(u64) * sq->wq_ctrl.buf.npages;
 892         in = mlx5_vzalloc(inlen);
 893         if (!in)
 894                 return -ENOMEM;
 895
 896         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 897         wq = MLX5_ADDR_OF(sqc, sqc, wq);
 898
 899         memcpy(sqc, param->sqc, sizeof(param->sqc));
 900
 901         MLX5_SET(sqc,  sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);
 902         MLX5_SET(sqc,  sqc, cqn,                sq->cq.mcq.cqn);
 903         MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);
 904         MLX5_SET(sqc,  sqc, state,              MLX5_SQC_STATE_RST);
 905         MLX5_SET(sqc,  sqc, tis_lst_sz,         param->icosq ? 0 : 1);
 906         MLX5_SET(sqc,  sqc, flush_in_error_en,  1);
 907
 908         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
 909         MLX5_SET(wq,   wq, uar_page,      sq->uar.index);
 910         MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
 911                                           MLX5_ADAPTER_PAGE_SHIFT);
 912         MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
 913
 914         mlx5_fill_page_array(&sq->wq_ctrl.buf,
 915                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 916
 917         err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
 918
 919         kvfree(in);
 920
 921         return err;
 922 }
 923
 924 static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
 925                            int next_state, bool update_rl, int rl_index)
 926 {
 927         struct mlx5e_channel *c = sq->channel;
 928         struct mlx5e_priv *priv = c->priv;
 929         struct mlx5_core_dev *mdev = priv->mdev;
 930
 931         void *in;
 932         void *sqc;
 933         int inlen;
 934         int err;
 935
 936         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
 937         in = mlx5_vzalloc(inlen);
 938         if (!in)
 939                 return -ENOMEM;
 940
 941         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
 942
 943         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
 944         MLX5_SET(sqc, sqc, state, next_state);
 945         if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
 946                 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
 947                 MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
 948         }
 949
 950         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
 951
 952         kvfree(in);
 953
 954         return err;
 955 }
 956
 957 static void mlx5e_disable_sq(struct mlx5e_sq *sq)
 958 {
 959         struct mlx5e_channel *c = sq->channel;
 960         struct mlx5e_priv *priv = c->priv;
 961         struct mlx5_core_dev *mdev = priv->mdev;
 962
 963         mlx5_core_destroy_sq(mdev, sq->sqn);
 964         if (sq->rate_limit)
 965                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
 966 }
 967
 968 static int mlx5e_open_sq(struct mlx5e_channel *c,
 969                          int tc,
 970                          struct mlx5e_sq_param *param,
 971                          struct mlx5e_sq *sq)
 972 {
 973         int err;
 974
 975         err = mlx5e_create_sq(c, tc, param, sq);
 976         if (err)
 977                 return err;
 978
 979         err = mlx5e_enable_sq(sq, param);
 980         if (err)
 981                 goto err_destroy_sq;
 982
 983         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
 984                               false, 0);
 985         if (err)
 986                 goto err_disable_sq;
 987
 988         if (sq->txq) {
 989                 netdev_tx_reset_queue(sq->txq);
 990                 netif_tx_start_queue(sq->txq);
 991         }
 992
 993         return 0;
 994
 995 err_disable_sq:
 996         mlx5e_disable_sq(sq);
 997 err_destroy_sq:
 998         mlx5e_destroy_sq(sq);
 999
1000         return err;
1001 }
1002
1003 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1004 {
1005         __netif_tx_lock_bh(txq);
1006         netif_tx_stop_queue(txq);
1007         __netif_tx_unlock_bh(txq);
1008 }
1009
1010 static void mlx5e_close_sq(struct mlx5e_sq *sq)
1011 {
1012         set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
1013         /* prevent netif_tx_wake_queue */
1014         napi_synchronize(&sq->channel->napi);
1015
1016         if (sq->txq) {
1017                 netif_tx_disable_queue(sq->txq);
1018
1019                 /* last doorbell out, godspeed .. */
1020                 if (mlx5e_sq_has_room_for(sq, 1))
1021                         mlx5e_send_nop(sq, true);
1022         }
1023
1024         mlx5e_disable_sq(sq);
1025         mlx5e_free_tx_descs(sq);
1026         mlx5e_destroy_sq(sq);
1027 }
1028
1029 static int mlx5e_create_cq(struct mlx5e_channel *c,
1030                            struct mlx5e_cq_param *param,
1031                            struct mlx5e_cq *cq)
1032 {
1033         struct mlx5e_priv *priv = c->priv;
1034         struct mlx5_core_dev *mdev = priv->mdev;
1035         struct mlx5_core_cq *mcq = &cq->mcq;
1036         int eqn_not_used;
1037         unsigned int irqn;
1038         int err;
1039         u32 i;
1040
1041         param->wq.buf_numa_node = cpu_to_node(c->cpu);
1042         param->wq.db_numa_node  = cpu_to_node(c->cpu);
1043         param->eq_ix   = c->ix;
1044
1045         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1046                                &cq->wq_ctrl);
1047         if (err)
1048                 return err;
1049
1050         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1051
1052         cq->napi        = &c->napi;
1053
1054         mcq->cqe_sz     = 64;
1055         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1056         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1057         *mcq->set_ci_db = 0;
1058         *mcq->arm_db    = 0;
1059         mcq->vector     = param->eq_ix;
1060         mcq->comp       = mlx5e_completion_event;
1061         mcq->event      = mlx5e_cq_error_event;
1062         mcq->irqn       = irqn;
1063         mcq->uar        = &mdev->mlx5e_res.cq_uar;
1064
1065         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1066                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1067
1068                 cqe->op_own = 0xf1;
1069         }
1070
1071         cq->channel = c;
1072         cq->priv = priv;
1073
1074         return 0;
1075 }
1076
1077 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1078 {
1079         mlx5_wq_destroy(&cq->wq_ctrl);
1080 }
1081
1082 static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1083 {
1084         struct mlx5e_priv *priv = cq->priv;
1085         struct mlx5_core_dev *mdev = priv->mdev;
1086         struct mlx5_core_cq *mcq = &cq->mcq;
1087
1088         void *in;
1089         void *cqc;
1090         int inlen;
1091         unsigned int irqn_not_used;
1092         int eqn;
1093         int err;
1094
1095         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1096                 sizeof(u64) * cq->wq_ctrl.buf.npages;
1097         in = mlx5_vzalloc(inlen);
1098         if (!in)
1099                 return -ENOMEM;
1100
1101         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1102
1103         memcpy(cqc, param->cqc, sizeof(param->cqc));
1104
1105         mlx5_fill_page_array(&cq->wq_ctrl.buf,
1106                              (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1107
1108         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1109
1110         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1111         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1112         MLX5_SET(cqc,   cqc, uar_page,      mcq->uar->index);
1113         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1114                                             MLX5_ADAPTER_PAGE_SHIFT);
1115         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1116
1117         err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1118
1119         kvfree(in);
1120
1121         if (err)
1122                 return err;
1123
1124         mlx5e_cq_arm(cq);
1125
1126         return 0;
1127 }
1128
1129 static void mlx5e_disable_cq(struct mlx5e_cq *cq)
1130 {
1131         struct mlx5e_priv *priv = cq->priv;
1132         struct mlx5_core_dev *mdev = priv->mdev;
1133
1134         mlx5_core_destroy_cq(mdev, &cq->mcq);
1135 }
1136
1137 static int mlx5e_open_cq(struct mlx5e_channel *c,
1138                          struct mlx5e_cq_param *param,
1139                          struct mlx5e_cq *cq,
1140                          struct mlx5e_cq_moder moderation)
1141 {
1142         int err;
1143         struct mlx5e_priv *priv = c->priv;
1144         struct mlx5_core_dev *mdev = priv->mdev;
1145
1146         err = mlx5e_create_cq(c, param, cq);
1147         if (err)
1148                 return err;
1149
1150         err = mlx5e_enable_cq(cq, param);
1151         if (err)
1152                 goto err_destroy_cq;
1153
1154         if (MLX5_CAP_GEN(mdev, cq_moderation))
1155                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
1156                                                moderation.usec,
1157                                                moderation.pkts);
1158         return 0;
1159
1160 err_destroy_cq:
1161         mlx5e_destroy_cq(cq);
1162
1163         return err;
1164 }
1165
1166 static void mlx5e_close_cq(struct mlx5e_cq *cq)
1167 {
1168         mlx5e_disable_cq(cq);
1169         mlx5e_destroy_cq(cq);
1170 }
1171
1172 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1173 {
1174         return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1175 }
1176
1177 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1178                              struct mlx5e_channel_param *cparam)
1179 {
1180         struct mlx5e_priv *priv = c->priv;
1181         int err;
1182         int tc;
1183
1184         for (tc = 0; tc < c->num_tc; tc++) {
1185                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
1186                                     priv->params.tx_cq_moderation);
1187                 if (err)
1188                         goto err_close_tx_cqs;
1189         }
1190
1191         return 0;
1192
1193 err_close_tx_cqs:
1194         for (tc--; tc >= 0; tc--)
1195                 mlx5e_close_cq(&c->sq[tc].cq);
1196
1197         return err;
1198 }
1199
1200 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1201 {
1202         int tc;
1203
1204         for (tc = 0; tc < c->num_tc; tc++)
1205                 mlx5e_close_cq(&c->sq[tc].cq);
1206 }
1207
1208 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1209                           struct mlx5e_channel_param *cparam)
1210 {
1211         int err;
1212         int tc;
1213
1214         for (tc = 0; tc < c->num_tc; tc++) {
1215                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1216                 if (err)
1217                         goto err_close_sqs;
1218         }
1219
1220         return 0;
1221
1222 err_close_sqs:
1223         for (tc--; tc >= 0; tc--)
1224                 mlx5e_close_sq(&c->sq[tc]);
1225
1226         return err;
1227 }
1228
1229 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1230 {
1231         int tc;
1232
1233         for (tc = 0; tc < c->num_tc; tc++)
1234                 mlx5e_close_sq(&c->sq[tc]);
1235 }
1236
1237 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
1238 {
1239         int i;
1240
1241         for (i = 0; i < priv->profile->max_tc; i++)
1242                 priv->channeltc_to_txq_map[ix][i] =
1243                         ix + i * priv->params.num_channels;
1244 }
1245
1246 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1247                                 struct mlx5e_sq *sq, u32 rate)
1248 {
1249         struct mlx5e_priv *priv = netdev_priv(dev);
1250         struct mlx5_core_dev *mdev = priv->mdev;
1251         u16 rl_index = 0;
1252         int err;
1253
1254         if (rate == sq->rate_limit)
1255                 /* nothing to do */
1256                 return 0;
1257
1258         if (sq->rate_limit)
1259                 /* remove current rl index to free space to next ones */
1260                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1261
1262         sq->rate_limit = 0;
1263
1264         if (rate) {
1265                 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1266                 if (err) {
1267                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1268                                    rate, err);
1269                         return err;
1270                 }
1271         }
1272
1273         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1274                               MLX5_SQC_STATE_RDY, true, rl_index);
1275         if (err) {
1276                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1277                            rate, err);
1278                 /* remove the rate from the table */
1279                 if (rate)
1280                         mlx5_rl_remove_rate(mdev, rate);
1281                 return err;
1282         }
1283
1284         sq->rate_limit = rate;
1285         return 0;
1286 }
1287
1288 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1289 {
1290         struct mlx5e_priv *priv = netdev_priv(dev);
1291         struct mlx5_core_dev *mdev = priv->mdev;
1292         struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1293         int err = 0;
1294
1295         if (!mlx5_rl_is_supported(mdev)) {
1296                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1297                 return -EINVAL;
1298         }
1299
1300         /* rate is given in Mb/sec, HW config is in Kb/sec */
1301         rate = rate << 10;
1302
1303         /* Check whether rate in valid range, 0 is always valid */
1304         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1305                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1306                 return -ERANGE;
1307         }
1308
1309         mutex_lock(&priv->state_lock);
1310         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1311                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1312         if (!err)
1313                 priv->tx_rates[index] = rate;
1314         mutex_unlock(&priv->state_lock);
1315
1316         return err;
1317 }
1318
1319 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1320                               struct mlx5e_channel_param *cparam,
1321                               struct mlx5e_channel **cp)
1322 {
1323         struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
1324         struct net_device *netdev = priv->netdev;
1325         struct mlx5e_cq_moder rx_cq_profile;
1326         int cpu = mlx5e_get_cpu(priv, ix);
1327         struct mlx5e_channel *c;
1328         struct mlx5e_sq *sq;
1329         int err;
1330         int i;
1331
1332         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1333         if (!c)
1334                 return -ENOMEM;
1335
1336         c->priv     = priv;
1337         c->ix       = ix;
1338         c->cpu      = cpu;
1339         c->pdev     = &priv->mdev->pdev->dev;
1340         c->netdev   = priv->netdev;
1341         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1342         c->num_tc   = priv->params.num_tc;
1343
1344         if (priv->params.rx_am_enabled)
1345                 rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1346         else
1347                 rx_cq_profile = priv->params.rx_cq_moderation;
1348
1349         mlx5e_build_channeltc_to_txq_map(priv, ix);
1350
1351         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1352
1353         err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
1354         if (err)
1355                 goto err_napi_del;
1356
1357         err = mlx5e_open_tx_cqs(c, cparam);
1358         if (err)
1359                 goto err_close_icosq_cq;
1360
1361         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1362                             rx_cq_profile);
1363         if (err)
1364                 goto err_close_tx_cqs;
1365
1366         napi_enable(&c->napi);
1367
1368         err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
1369         if (err)
1370                 goto err_disable_napi;
1371
1372         err = mlx5e_open_sqs(c, cparam);
1373         if (err)
1374                 goto err_close_icosq;
1375
1376         for (i = 0; i < priv->params.num_tc; i++) {
1377                 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1378
1379                 if (priv->tx_rates[txq_ix]) {
1380                         sq = priv->txq_to_sq_map[txq_ix];
1381                         mlx5e_set_sq_maxrate(priv->netdev, sq,
1382                                              priv->tx_rates[txq_ix]);
1383                 }
1384         }
1385
1386         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1387         if (err)
1388                 goto err_close_sqs;
1389
1390         netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1391         *cp = c;
1392
1393         return 0;
1394
1395 err_close_sqs:
1396         mlx5e_close_sqs(c);
1397
1398 err_close_icosq:
1399         mlx5e_close_sq(&c->icosq);
1400
1401 err_disable_napi:
1402         napi_disable(&c->napi);
1403         mlx5e_close_cq(&c->rq.cq);
1404
1405 err_close_tx_cqs:
1406         mlx5e_close_tx_cqs(c);
1407
1408 err_close_icosq_cq:
1409         mlx5e_close_cq(&c->icosq.cq);
1410
1411 err_napi_del:
1412         netif_napi_del(&c->napi);
1413         napi_hash_del(&c->napi);
1414         kfree(c);
1415
1416         return err;
1417 }
1418
1419 static void mlx5e_close_channel(struct mlx5e_channel *c)
1420 {
1421         mlx5e_close_rq(&c->rq);
1422         mlx5e_close_sqs(c);
1423         mlx5e_close_sq(&c->icosq);
1424         napi_disable(&c->napi);
1425         mlx5e_close_cq(&c->rq.cq);
1426         mlx5e_close_tx_cqs(c);
1427         mlx5e_close_cq(&c->icosq.cq);
1428         netif_napi_del(&c->napi);
1429
1430         napi_hash_del(&c->napi);
1431         synchronize_rcu();
1432
1433         kfree(c);
1434 }
1435
1436 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1437                                  struct mlx5e_rq_param *param)
1438 {
1439         void *rqc = param->rqc;
1440         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1441
1442         switch (priv->params.rq_wq_type) {
1443         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1444                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1445                          priv->params.mpwqe_log_num_strides - 9);
1446                 MLX5_SET(wq, wq, log_wqe_stride_size,
1447                          priv->params.mpwqe_log_stride_sz - 6);
1448                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1449                 break;
1450         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1451                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1452         }
1453
1454         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1455         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1456         MLX5_SET(wq, wq, log_wq_sz,        priv->params.log_rq_size);
1457         MLX5_SET(wq, wq, pd,               priv->mdev->mlx5e_res.pdn);
1458         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1459
1460         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1461         param->wq.linear = 1;
1462
1463         param->am_enabled = priv->params.rx_am_enabled;
1464 }
1465
1466 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1467 {
1468         void *rqc = param->rqc;
1469         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1470
1471         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1472         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1473 }
1474
1475 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1476                                         struct mlx5e_sq_param *param)
1477 {
1478         void *sqc = param->sqc;
1479         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1480
1481         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1482         MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
1483
1484         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1485 }
1486
1487 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1488                                  struct mlx5e_sq_param *param)
1489 {
1490         void *sqc = param->sqc;
1491         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1492
1493         mlx5e_build_sq_param_common(priv, param);
1494         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1495
1496         param->max_inline = priv->params.tx_max_inline;
1497         param->min_inline_mode = priv->params.tx_min_inline_mode;
1498 }
1499
1500 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1501                                         struct mlx5e_cq_param *param)
1502 {
1503         void *cqc = param->cqc;
1504
1505         MLX5_SET(cqc, cqc, uar_page, priv->mdev->mlx5e_res.cq_uar.index);
1506 }
1507
1508 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1509                                     struct mlx5e_cq_param *param)
1510 {
1511         void *cqc = param->cqc;
1512         u8 log_cq_size;
1513
1514         switch (priv->params.rq_wq_type) {
1515         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1516                 log_cq_size = priv->params.log_rq_size +
1517                         priv->params.mpwqe_log_num_strides;
1518                 break;
1519         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1520                 log_cq_size = priv->params.log_rq_size;
1521         }
1522
1523         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
1524         if (priv->params.rx_cqe_compress) {
1525                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1526                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1527         }
1528
1529         mlx5e_build_common_cq_param(priv, param);
1530
1531         param->cq_period_mode = priv->params.rx_cq_period_mode;
1532 }
1533
1534 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1535                                     struct mlx5e_cq_param *param)
1536 {
1537         void *cqc = param->cqc;
1538
1539         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1540
1541         mlx5e_build_common_cq_param(priv, param);
1542
1543         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1544 }
1545
1546 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1547                                      struct mlx5e_cq_param *param,
1548                                      u8 log_wq_size)
1549 {
1550         void *cqc = param->cqc;
1551
1552         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1553
1554         mlx5e_build_common_cq_param(priv, param);
1555
1556         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1557 }
1558
1559 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1560                                     struct mlx5e_sq_param *param,
1561                                     u8 log_wq_size)
1562 {
1563         void *sqc = param->sqc;
1564         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1565
1566         mlx5e_build_sq_param_common(priv, param);
1567
1568         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1569         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
1570
1571         param->icosq = true;
1572 }
1573
1574 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
1575 {
1576         u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1577
1578         mlx5e_build_rq_param(priv, &cparam->rq);
1579         mlx5e_build_sq_param(priv, &cparam->sq);
1580         mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
1581         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1582         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1583         mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
1584 }
1585
1586 static int mlx5e_open_channels(struct mlx5e_priv *priv)
1587 {
1588         struct mlx5e_channel_param *cparam;
1589         int nch = priv->params.num_channels;
1590         int err = -ENOMEM;
1591         int i;
1592         int j;
1593
1594         priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1595                                 GFP_KERNEL);
1596
1597         priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
1598                                       sizeof(struct mlx5e_sq *), GFP_KERNEL);
1599
1600         cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1601
1602         if (!priv->channel || !priv->txq_to_sq_map || !cparam)
1603                 goto err_free_txq_to_sq_map;
1604
1605         mlx5e_build_channel_param(priv, cparam);
1606
1607         for (i = 0; i < nch; i++) {
1608                 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
1609                 if (err)
1610                         goto err_close_channels;
1611         }
1612
1613         for (j = 0; j < nch; j++) {
1614                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1615                 if (err)
1616                         goto err_close_channels;
1617         }
1618
1619         /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1620          * polling for inactive tx queues.
1621          */
1622         netif_tx_start_all_queues(priv->netdev);
1623
1624         kfree(cparam);
1625         return 0;
1626
1627 err_close_channels:
1628         for (i--; i >= 0; i--)
1629                 mlx5e_close_channel(priv->channel[i]);
1630
1631 err_free_txq_to_sq_map:
1632         kfree(priv->txq_to_sq_map);
1633         kfree(priv->channel);
1634         kfree(cparam);
1635
1636         return err;
1637 }
1638
1639 static void mlx5e_close_channels(struct mlx5e_priv *priv)
1640 {
1641         int i;
1642
1643         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1644          * polling for inactive tx queues.
1645          */
1646         netif_tx_stop_all_queues(priv->netdev);
1647         netif_tx_disable(priv->netdev);
1648
1649         for (i = 0; i < priv->params.num_channels; i++)
1650                 mlx5e_close_channel(priv->channel[i]);
1651
1652         kfree(priv->txq_to_sq_map);
1653         kfree(priv->channel);
1654 }
1655
1656 static int mlx5e_rx_hash_fn(int hfunc)
1657 {
1658         return (hfunc == ETH_RSS_HASH_TOP) ?
1659                MLX5_RX_HASH_FN_TOEPLITZ :
1660                MLX5_RX_HASH_FN_INVERTED_XOR8;
1661 }
1662
1663 static int mlx5e_bits_invert(unsigned long a, int size)
1664 {
1665         int inv = 0;
1666         int i;
1667
1668         for (i = 0; i < size; i++)
1669                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1670
1671         return inv;
1672 }
1673
1674 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1675 {
1676         int i;
1677
1678         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1679                 int ix = i;
1680                 u32 rqn;
1681
1682                 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1683                         ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1684
1685                 ix = priv->params.indirection_rqt[ix];
1686                 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1687                                 priv->channel[ix]->rq.rqn :
1688                                 priv->drop_rq.rqn;
1689                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
1690         }
1691 }
1692
1693 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1694                                       int ix)
1695 {
1696         u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1697                         priv->channel[ix]->rq.rqn :
1698                         priv->drop_rq.rqn;
1699
1700         MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
1701 }
1702
1703 static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
1704                             int ix, struct mlx5e_rqt *rqt)
1705 {
1706         struct mlx5_core_dev *mdev = priv->mdev;
1707         void *rqtc;
1708         int inlen;
1709         int err;
1710         u32 *in;
1711
1712         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1713         in = mlx5_vzalloc(inlen);
1714         if (!in)
1715                 return -ENOMEM;
1716
1717         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1718
1719         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1720         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1721
1722         if (sz > 1) /* RSS */
1723                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1724         else
1725                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1726
1727         err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
1728         if (!err)
1729                 rqt->enabled = true;
1730
1731         kvfree(in);
1732         return err;
1733 }
1734
1735 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1736 {
1737         rqt->enabled = false;
1738         mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1739 }
1740
1741 static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
1742 {
1743         struct mlx5e_rqt *rqt = &priv->indir_rqt;
1744
1745         return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
1746 }
1747
1748 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1749 {
1750         struct mlx5e_rqt *rqt;
1751         int err;
1752         int ix;
1753
1754         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1755                 rqt = &priv->direct_tir[ix].rqt;
1756                 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
1757                 if (err)
1758                         goto err_destroy_rqts;
1759         }
1760
1761         return 0;
1762
1763 err_destroy_rqts:
1764         for (ix--; ix >= 0; ix--)
1765                 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1766
1767         return err;
1768 }
1769
1770 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
1771 {
1772         struct mlx5_core_dev *mdev = priv->mdev;
1773         void *rqtc;
1774         int inlen;
1775         u32 *in;
1776         int err;
1777
1778         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1779         in = mlx5_vzalloc(inlen);
1780         if (!in)
1781                 return -ENOMEM;
1782
1783         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1784
1785         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1786         if (sz > 1) /* RSS */
1787                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1788         else
1789                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1790
1791         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1792
1793         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
1794
1795         kvfree(in);
1796
1797         return err;
1798 }
1799
1800 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1801 {
1802         u32 rqtn;
1803         int ix;
1804
1805         if (priv->indir_rqt.enabled) {
1806                 rqtn = priv->indir_rqt.rqtn;
1807                 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
1808         }
1809
1810         for (ix = 0; ix < priv->params.num_channels; ix++) {
1811                 if (!priv->direct_tir[ix].rqt.enabled)
1812                         continue;
1813                 rqtn = priv->direct_tir[ix].rqt.rqtn;
1814                 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
1815         }
1816 }
1817
1818 static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
1819 {
1820         if (!priv->params.lro_en)
1821                 return;
1822
1823 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1824
1825         MLX5_SET(tirc, tirc, lro_enable_mask,
1826                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1827                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1828         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
1829                  (priv->params.lro_wqe_sz -
1830                   ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1831         MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1832                  MLX5_CAP_ETH(priv->mdev,
1833                               lro_timer_supported_periods[2]));
1834 }
1835
1836 void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
1837 {
1838         MLX5_SET(tirc, tirc, rx_hash_fn,
1839                  mlx5e_rx_hash_fn(priv->params.rss_hfunc));
1840         if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
1841                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
1842                                              rx_hash_toeplitz_key);
1843                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
1844                                                rx_hash_toeplitz_key);
1845
1846                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1847                 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
1848         }
1849 }
1850
1851 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
1852 {
1853         struct mlx5_core_dev *mdev = priv->mdev;
1854
1855         void *in;
1856         void *tirc;
1857         int inlen;
1858         int err;
1859         int tt;
1860         int ix;
1861
1862         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1863         in = mlx5_vzalloc(inlen);
1864         if (!in)
1865                 return -ENOMEM;
1866
1867         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
1868         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
1869
1870         mlx5e_build_tir_ctx_lro(tirc, priv);
1871
1872         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
1873                 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
1874                                            inlen);
1875                 if (err)
1876                         goto free_in;
1877         }
1878
1879         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1880                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
1881                                            in, inlen);
1882                 if (err)
1883                         goto free_in;
1884         }
1885
1886 free_in:
1887         kvfree(in);
1888
1889         return err;
1890 }
1891
1892 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
1893 {
1894         struct mlx5_core_dev *mdev = priv->mdev;
1895         u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
1896         int err;
1897
1898         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
1899         if (err)
1900                 return err;
1901
1902         /* Update vport context MTU */
1903         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
1904         return 0;
1905 }
1906
1907 static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
1908 {
1909         struct mlx5_core_dev *mdev = priv->mdev;
1910         u16 hw_mtu = 0;
1911         int err;
1912
1913         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
1914         if (err || !hw_mtu) /* fallback to port oper mtu */
1915                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
1916
1917         *mtu = MLX5E_HW2SW_MTU(hw_mtu);
1918 }
1919
1920 static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
1921 {
1922         struct mlx5e_priv *priv = netdev_priv(netdev);
1923         u16 mtu;
1924         int err;
1925
1926         err = mlx5e_set_mtu(priv, netdev->mtu);
1927         if (err)
1928                 return err;
1929
1930         mlx5e_query_mtu(priv, &mtu);
1931         if (mtu != netdev->mtu)
1932                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
1933                             __func__, mtu, netdev->mtu);
1934
1935         netdev->mtu = mtu;
1936         return 0;
1937 }
1938
1939 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
1940 {
1941         struct mlx5e_priv *priv = netdev_priv(netdev);
1942         int nch = priv->params.num_channels;
1943         int ntc = priv->params.num_tc;
1944         int tc;
1945
1946         netdev_reset_tc(netdev);
1947
1948         if (ntc == 1)
1949                 return;
1950
1951         netdev_set_num_tc(netdev, ntc);
1952
1953         /* Map netdev TCs to offset 0
1954          * We have our own UP to TXQ mapping for QoS
1955          */
1956         for (tc = 0; tc < ntc; tc++)
1957                 netdev_set_tc_queue(netdev, tc, nch, 0);
1958 }
1959
1960 int mlx5e_open_locked(struct net_device *netdev)
1961 {
1962         struct mlx5e_priv *priv = netdev_priv(netdev);
1963         struct mlx5_core_dev *mdev = priv->mdev;
1964         int num_txqs;
1965         int err;
1966
1967         set_bit(MLX5E_STATE_OPENED, &priv->state);
1968
1969         mlx5e_netdev_set_tcs(netdev);
1970
1971         num_txqs = priv->params.num_channels * priv->params.num_tc;
1972         netif_set_real_num_tx_queues(netdev, num_txqs);
1973         netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
1974
1975         err = mlx5e_open_channels(priv);
1976         if (err) {
1977                 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
1978                            __func__, err);
1979                 goto err_clear_state_opened_flag;
1980         }
1981
1982         err = mlx5e_refresh_tirs_self_loopback_enable(priv->mdev);
1983         if (err) {
1984                 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
1985                            __func__, err);
1986                 goto err_close_channels;
1987         }
1988
1989         mlx5e_redirect_rqts(priv);
1990         mlx5e_update_carrier(priv);
1991         mlx5e_timestamp_init(priv);
1992 #ifdef CONFIG_RFS_ACCEL
1993         priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
1994 #endif
1995         if (priv->profile->update_stats)
1996                 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
1997
1998         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
1999                 err = mlx5e_add_sqs_fwd_rules(priv);
2000                 if (err)
2001                         goto err_close_channels;
2002         }
2003         return 0;
2004
2005 err_close_channels:
2006         mlx5e_close_channels(priv);
2007 err_clear_state_opened_flag:
2008         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2009         return err;
2010 }
2011
2012 int mlx5e_open(struct net_device *netdev)
2013 {
2014         struct mlx5e_priv *priv = netdev_priv(netdev);
2015         int err;
2016
2017         mutex_lock(&priv->state_lock);
2018         err = mlx5e_open_locked(netdev);
2019         mutex_unlock(&priv->state_lock);
2020
2021         return err;
2022 }
2023
2024 int mlx5e_close_locked(struct net_device *netdev)
2025 {
2026         struct mlx5e_priv *priv = netdev_priv(netdev);
2027         struct mlx5_core_dev *mdev = priv->mdev;
2028
2029         /* May already be CLOSED in case a previous configuration operation
2030          * (e.g RX/TX queue size change) that involves close&open failed.
2031          */
2032         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2033                 return 0;
2034
2035         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2036
2037         if (MLX5_CAP_GEN(mdev, vport_group_manager))
2038                 mlx5e_remove_sqs_fwd_rules(priv);
2039
2040         mlx5e_timestamp_cleanup(priv);
2041         netif_carrier_off(priv->netdev);
2042         mlx5e_redirect_rqts(priv);
2043         mlx5e_close_channels(priv);
2044
2045         return 0;
2046 }
2047
2048 int mlx5e_close(struct net_device *netdev)
2049 {
2050         struct mlx5e_priv *priv = netdev_priv(netdev);
2051         int err;
2052
2053         if (!netif_device_present(netdev))
2054                 return -ENODEV;
2055
2056         mutex_lock(&priv->state_lock);
2057         err = mlx5e_close_locked(netdev);
2058         mutex_unlock(&priv->state_lock);
2059
2060         return err;
2061 }
2062
2063 static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
2064                                 struct mlx5e_rq *rq,
2065                                 struct mlx5e_rq_param *param)
2066 {
2067         struct mlx5_core_dev *mdev = priv->mdev;
2068         void *rqc = param->rqc;
2069         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2070         int err;
2071
2072         param->wq.db_numa_node = param->wq.buf_numa_node;
2073
2074         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2075                                 &rq->wq_ctrl);
2076         if (err)
2077                 return err;
2078
2079         rq->priv = priv;
2080
2081         return 0;
2082 }
2083
2084 static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2085                                 struct mlx5e_cq *cq,
2086                                 struct mlx5e_cq_param *param)
2087 {
2088         struct mlx5_core_dev *mdev = priv->mdev;
2089         struct mlx5_core_cq *mcq = &cq->mcq;
2090         int eqn_not_used;
2091         unsigned int irqn;
2092         int err;
2093
2094         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2095                                &cq->wq_ctrl);
2096         if (err)
2097                 return err;
2098
2099         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
2100
2101         mcq->cqe_sz     = 64;
2102         mcq->set_ci_db  = cq->wq_ctrl.db.db;
2103         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
2104         *mcq->set_ci_db = 0;
2105         *mcq->arm_db    = 0;
2106         mcq->vector     = param->eq_ix;
2107         mcq->comp       = mlx5e_completion_event;
2108         mcq->event      = mlx5e_cq_error_event;
2109         mcq->irqn       = irqn;
2110         mcq->uar        = &mdev->mlx5e_res.cq_uar;
2111
2112         cq->priv = priv;
2113
2114         return 0;
2115 }
2116
2117 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
2118 {
2119         struct mlx5e_cq_param cq_param;
2120         struct mlx5e_rq_param rq_param;
2121         struct mlx5e_rq *rq = &priv->drop_rq;
2122         struct mlx5e_cq *cq = &priv->drop_rq.cq;
2123         int err;
2124
2125         memset(&cq_param, 0, sizeof(cq_param));
2126         memset(&rq_param, 0, sizeof(rq_param));
2127         mlx5e_build_drop_rq_param(&rq_param);
2128
2129         err = mlx5e_create_drop_cq(priv, cq, &cq_param);
2130         if (err)
2131                 return err;
2132
2133         err = mlx5e_enable_cq(cq, &cq_param);
2134         if (err)
2135                 goto err_destroy_cq;
2136
2137         err = mlx5e_create_drop_rq(priv, rq, &rq_param);
2138         if (err)
2139                 goto err_disable_cq;
2140
2141         err = mlx5e_enable_rq(rq, &rq_param);
2142         if (err)
2143                 goto err_destroy_rq;
2144
2145         return 0;
2146
2147 err_destroy_rq:
2148         mlx5e_destroy_rq(&priv->drop_rq);
2149
2150 err_disable_cq:
2151         mlx5e_disable_cq(&priv->drop_rq.cq);
2152
2153 err_destroy_cq:
2154         mlx5e_destroy_cq(&priv->drop_rq.cq);
2155
2156         return err;
2157 }
2158
2159 static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
2160 {
2161         mlx5e_disable_rq(&priv->drop_rq);
2162         mlx5e_destroy_rq(&priv->drop_rq);
2163         mlx5e_disable_cq(&priv->drop_rq.cq);
2164         mlx5e_destroy_cq(&priv->drop_rq.cq);
2165 }
2166
2167 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
2168 {
2169         struct mlx5_core_dev *mdev = priv->mdev;
2170         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
2171         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2172
2173         MLX5_SET(tisc, tisc, prio, tc << 1);
2174         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
2175
2176         if (mlx5_lag_is_lacp_owner(mdev))
2177                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2178
2179         return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
2180 }
2181
2182 static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
2183 {
2184         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2185 }
2186
2187 int mlx5e_create_tises(struct mlx5e_priv *priv)
2188 {
2189         int err;
2190         int tc;
2191
2192         for (tc = 0; tc < priv->profile->max_tc; tc++) {
2193                 err = mlx5e_create_tis(priv, tc);
2194                 if (err)
2195                         goto err_close_tises;
2196         }
2197
2198         return 0;
2199
2200 err_close_tises:
2201         for (tc--; tc >= 0; tc--)
2202                 mlx5e_destroy_tis(priv, tc);
2203
2204         return err;
2205 }
2206
2207 void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
2208 {
2209         int tc;
2210
2211         for (tc = 0; tc < priv->profile->max_tc; tc++)
2212                 mlx5e_destroy_tis(priv, tc);
2213 }
2214
2215 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2216                                       enum mlx5e_traffic_types tt)
2217 {
2218         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2219
2220         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2221
2222 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2223                                  MLX5_HASH_FIELD_SEL_DST_IP)
2224
2225 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2226                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2227                                  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2228                                  MLX5_HASH_FIELD_SEL_L4_DPORT)
2229
2230 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2231                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2232                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2233
2234         mlx5e_build_tir_ctx_lro(tirc, priv);
2235
2236         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2237         MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2238         mlx5e_build_tir_ctx_hash(tirc, priv);
2239
2240         switch (tt) {
2241         case MLX5E_TT_IPV4_TCP:
2242                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2243                          MLX5_L3_PROT_TYPE_IPV4);
2244                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2245                          MLX5_L4_PROT_TYPE_TCP);
2246                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2247                          MLX5_HASH_IP_L4PORTS);
2248                 break;
2249
2250         case MLX5E_TT_IPV6_TCP:
2251                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2252                          MLX5_L3_PROT_TYPE_IPV6);
2253                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2254                          MLX5_L4_PROT_TYPE_TCP);
2255                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2256                          MLX5_HASH_IP_L4PORTS);
2257                 break;
2258
2259         case MLX5E_TT_IPV4_UDP:
2260                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2261                          MLX5_L3_PROT_TYPE_IPV4);
2262                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2263                          MLX5_L4_PROT_TYPE_UDP);
2264                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2265                          MLX5_HASH_IP_L4PORTS);
2266                 break;
2267
2268         case MLX5E_TT_IPV6_UDP:
2269                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2270                          MLX5_L3_PROT_TYPE_IPV6);
2271                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2272                          MLX5_L4_PROT_TYPE_UDP);
2273                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2274                          MLX5_HASH_IP_L4PORTS);
2275                 break;
2276
2277         case MLX5E_TT_IPV4_IPSEC_AH:
2278                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2279                          MLX5_L3_PROT_TYPE_IPV4);
2280                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2281                          MLX5_HASH_IP_IPSEC_SPI);
2282                 break;
2283
2284         case MLX5E_TT_IPV6_IPSEC_AH:
2285                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2286                          MLX5_L3_PROT_TYPE_IPV6);
2287                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2288                          MLX5_HASH_IP_IPSEC_SPI);
2289                 break;
2290
2291         case MLX5E_TT_IPV4_IPSEC_ESP:
2292                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2293                          MLX5_L3_PROT_TYPE_IPV4);
2294                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2295                          MLX5_HASH_IP_IPSEC_SPI);
2296                 break;
2297
2298         case MLX5E_TT_IPV6_IPSEC_ESP:
2299                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2300                          MLX5_L3_PROT_TYPE_IPV6);
2301                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2302                          MLX5_HASH_IP_IPSEC_SPI);
2303                 break;
2304
2305         case MLX5E_TT_IPV4:
2306                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2307                          MLX5_L3_PROT_TYPE_IPV4);
2308                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2309                          MLX5_HASH_IP);
2310                 break;
2311
2312         case MLX5E_TT_IPV6:
2313                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2314                          MLX5_L3_PROT_TYPE_IPV6);
2315                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2316                          MLX5_HASH_IP);
2317                 break;
2318         default:
2319                 WARN_ONCE(true,
2320                           "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2321         }
2322 }
2323
2324 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2325                                        u32 rqtn)
2326 {
2327         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2328
2329         mlx5e_build_tir_ctx_lro(tirc, priv);
2330
2331         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2332         MLX5_SET(tirc, tirc, indirect_table, rqtn);
2333         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2334 }
2335
2336 static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
2337 {
2338         struct mlx5e_tir *tir;
2339         void *tirc;
2340         int inlen;
2341         int err;
2342         u32 *in;
2343         int tt;
2344
2345         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2346         in = mlx5_vzalloc(inlen);
2347         if (!in)
2348                 return -ENOMEM;
2349
2350         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2351                 memset(in, 0, inlen);
2352                 tir = &priv->indir_tir[tt];
2353                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2354                 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
2355                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2356                 if (err)
2357                         goto err_destroy_tirs;
2358         }
2359
2360         kvfree(in);
2361
2362         return 0;
2363
2364 err_destroy_tirs:
2365         for (tt--; tt >= 0; tt--)
2366                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2367
2368         kvfree(in);
2369
2370         return err;
2371 }
2372
2373 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
2374 {
2375         int nch = priv->profile->max_nch(priv->mdev);
2376         struct mlx5e_tir *tir;
2377         void *tirc;
2378         int inlen;
2379         int err;
2380         u32 *in;
2381         int ix;
2382
2383         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2384         in = mlx5_vzalloc(inlen);
2385         if (!in)
2386                 return -ENOMEM;
2387
2388         for (ix = 0; ix < nch; ix++) {
2389                 memset(in, 0, inlen);
2390                 tir = &priv->direct_tir[ix];
2391                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2392                 mlx5e_build_direct_tir_ctx(priv, tirc,
2393                                            priv->direct_tir[ix].rqt.rqtn);
2394                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2395                 if (err)
2396                         goto err_destroy_ch_tirs;
2397         }
2398
2399         kvfree(in);
2400
2401         return 0;
2402
2403 err_destroy_ch_tirs:
2404         for (ix--; ix >= 0; ix--)
2405                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
2406
2407         kvfree(in);
2408
2409         return err;
2410 }
2411
2412 static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
2413 {
2414         int i;
2415
2416         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
2417                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
2418 }
2419
2420 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
2421 {
2422         int nch = priv->profile->max_nch(priv->mdev);
2423         int i;
2424
2425         for (i = 0; i < nch; i++)
2426                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2427 }
2428
2429 int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2430 {
2431         int err = 0;
2432         int i;
2433
2434         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2435                 return 0;
2436
2437         for (i = 0; i < priv->params.num_channels; i++) {
2438                 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2439                 if (err)
2440                         return err;
2441         }
2442
2443         return 0;
2444 }
2445
2446 static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2447 {
2448         struct mlx5e_priv *priv = netdev_priv(netdev);
2449         bool was_opened;
2450         int err = 0;
2451
2452         if (tc && tc != MLX5E_MAX_NUM_TC)
2453                 return -EINVAL;
2454
2455         mutex_lock(&priv->state_lock);
2456
2457         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2458         if (was_opened)
2459                 mlx5e_close_locked(priv->netdev);
2460
2461         priv->params.num_tc = tc ? tc : 1;
2462
2463         if (was_opened)
2464                 err = mlx5e_open_locked(priv->netdev);
2465
2466         mutex_unlock(&priv->state_lock);
2467
2468         return err;
2469 }
2470
2471 static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2472                               __be16 proto, struct tc_to_netdev *tc)
2473 {
2474         struct mlx5e_priv *priv = netdev_priv(dev);
2475
2476         if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2477                 goto mqprio;
2478
2479         switch (tc->type) {
2480         case TC_SETUP_CLSFLOWER:
2481                 switch (tc->cls_flower->command) {
2482                 case TC_CLSFLOWER_REPLACE:
2483                         return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2484                 case TC_CLSFLOWER_DESTROY:
2485                         return mlx5e_delete_flower(priv, tc->cls_flower);
2486                 case TC_CLSFLOWER_STATS:
2487                         return mlx5e_stats_flower(priv, tc->cls_flower);
2488                 }
2489         default:
2490                 return -EOPNOTSUPP;
2491         }
2492
2493 mqprio:
2494         if (tc->type != TC_SETUP_MQPRIO)
2495                 return -EINVAL;
2496
2497         return mlx5e_setup_tc(dev, tc->tc);
2498 }
2499
2500 struct rtnl_link_stats64 *
2501 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2502 {
2503         struct mlx5e_priv *priv = netdev_priv(dev);
2504         struct mlx5e_sw_stats *sstats = &priv->stats.sw;
2505         struct mlx5e_vport_stats *vstats = &priv->stats.vport;
2506         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
2507
2508         stats->rx_packets = sstats->rx_packets;
2509         stats->rx_bytes   = sstats->rx_bytes;
2510         stats->tx_packets = sstats->tx_packets;
2511         stats->tx_bytes   = sstats->tx_bytes;
2512
2513         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
2514         stats->tx_dropped = sstats->tx_queue_dropped;
2515
2516         stats->rx_length_errors =
2517                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2518                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2519                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
2520         stats->rx_crc_errors =
2521                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2522         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2523         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
2524         stats->tx_carrier_errors =
2525                 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
2526         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2527                            stats->rx_frame_errors;
2528         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2529
2530         /* vport multicast also counts packets that are dropped due to steering
2531          * or rx out of buffer
2532          */
2533         stats->multicast =
2534                 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
2535
2536         return stats;
2537 }
2538
2539 static void mlx5e_set_rx_mode(struct net_device *dev)
2540 {
2541         struct mlx5e_priv *priv = netdev_priv(dev);
2542
2543         queue_work(priv->wq, &priv->set_rx_mode_work);
2544 }
2545
2546 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2547 {
2548         struct mlx5e_priv *priv = netdev_priv(netdev);
2549         struct sockaddr *saddr = addr;
2550
2551         if (!is_valid_ether_addr(saddr->sa_data))
2552                 return -EADDRNOTAVAIL;
2553
2554         netif_addr_lock_bh(netdev);
2555         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2556         netif_addr_unlock_bh(netdev);
2557
2558         queue_work(priv->wq, &priv->set_rx_mode_work);
2559
2560         return 0;
2561 }
2562
2563 #define MLX5E_SET_FEATURE(netdev, feature, enable)      \
2564         do {                                            \
2565                 if (enable)                             \
2566                         netdev->features |= feature;    \
2567                 else                                    \
2568                         netdev->features &= ~feature;   \
2569         } while (0)
2570
2571 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2572
2573 static int set_feature_lro(struct net_device *netdev, bool enable)
2574 {
2575         struct mlx5e_priv *priv = netdev_priv(netdev);
2576         bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2577         int err;
2578
2579         mutex_lock(&priv->state_lock);
2580
2581         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2582                 mlx5e_close_locked(priv->netdev);
2583
2584         priv->params.lro_en = enable;
2585         err = mlx5e_modify_tirs_lro(priv);
2586         if (err) {
2587                 netdev_err(netdev, "lro modify failed, %d\n", err);
2588                 priv->params.lro_en = !enable;
2589         }
2590
2591         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2592                 mlx5e_open_locked(priv->netdev);
2593
2594         mutex_unlock(&priv->state_lock);
2595
2596         return err;
2597 }
2598
2599 static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2600 {
2601         struct mlx5e_priv *priv = netdev_priv(netdev);
2602
2603         if (enable)
2604                 mlx5e_enable_vlan_filter(priv);
2605         else
2606                 mlx5e_disable_vlan_filter(priv);
2607
2608         return 0;
2609 }
2610
2611 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2612 {
2613         struct mlx5e_priv *priv = netdev_priv(netdev);
2614
2615         if (!enable && mlx5e_tc_num_filters(priv)) {
2616                 netdev_err(netdev,
2617                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2618                 return -EINVAL;
2619         }
2620
2621         return 0;
2622 }
2623
2624 static int set_feature_rx_all(struct net_device *netdev, bool enable)
2625 {
2626         struct mlx5e_priv *priv = netdev_priv(netdev);
2627         struct mlx5_core_dev *mdev = priv->mdev;
2628
2629         return mlx5_set_port_fcs(mdev, !enable);
2630 }
2631
2632 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2633 {
2634         struct mlx5e_priv *priv = netdev_priv(netdev);
2635         int err;
2636
2637         mutex_lock(&priv->state_lock);
2638
2639         priv->params.vlan_strip_disable = !enable;
2640         err = mlx5e_modify_rqs_vsd(priv, !enable);
2641         if (err)
2642                 priv->params.vlan_strip_disable = enable;
2643
2644         mutex_unlock(&priv->state_lock);
2645
2646         return err;
2647 }
2648
2649 #ifdef CONFIG_RFS_ACCEL
2650 static int set_feature_arfs(struct net_device *netdev, bool enable)
2651 {
2652         struct mlx5e_priv *priv = netdev_priv(netdev);
2653         int err;
2654
2655         if (enable)
2656                 err = mlx5e_arfs_enable(priv);
2657         else
2658                 err = mlx5e_arfs_disable(priv);
2659
2660         return err;
2661 }
2662 #endif
2663
2664 static int mlx5e_handle_feature(struct net_device *netdev,
2665                                 netdev_features_t wanted_features,
2666                                 netdev_features_t feature,
2667                                 mlx5e_feature_handler feature_handler)
2668 {
2669         netdev_features_t changes = wanted_features ^ netdev->features;
2670         bool enable = !!(wanted_features & feature);
2671         int err;
2672
2673         if (!(changes & feature))
2674                 return 0;
2675
2676         err = feature_handler(netdev, enable);
2677         if (err) {
2678                 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2679                            enable ? "Enable" : "Disable", feature, err);
2680                 return err;
2681         }
2682
2683         MLX5E_SET_FEATURE(netdev, feature, enable);
2684         return 0;
2685 }
2686
2687 static int mlx5e_set_features(struct net_device *netdev,
2688                               netdev_features_t features)
2689 {
2690         int err;
2691
2692         err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2693                                     set_feature_lro);
2694         err |= mlx5e_handle_feature(netdev, features,
2695                                     NETIF_F_HW_VLAN_CTAG_FILTER,
2696                                     set_feature_vlan_filter);
2697         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2698                                     set_feature_tc_num_filters);
2699         err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2700                                     set_feature_rx_all);
2701         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2702                                     set_feature_rx_vlan);
2703 #ifdef CONFIG_RFS_ACCEL
2704         err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2705                                     set_feature_arfs);
2706 #endif
2707
2708         return err ? -EINVAL : 0;
2709 }
2710
2711 #define MXL5_HW_MIN_MTU 64
2712 #define MXL5E_MIN_MTU (MXL5_HW_MIN_MTU + ETH_FCS_LEN)
2713
2714 static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2715 {
2716         struct mlx5e_priv *priv = netdev_priv(netdev);
2717         struct mlx5_core_dev *mdev = priv->mdev;
2718         bool was_opened;
2719         u16 max_mtu;
2720         u16 min_mtu;
2721         int err = 0;
2722         bool reset;
2723
2724         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2725
2726         max_mtu = MLX5E_HW2SW_MTU(max_mtu);
2727         min_mtu = MLX5E_HW2SW_MTU(MXL5E_MIN_MTU);
2728
2729         if (new_mtu > max_mtu || new_mtu < min_mtu) {
2730                 netdev_err(netdev,
2731                            "%s: Bad MTU (%d), valid range is: [%d..%d]\n",
2732                            __func__, new_mtu, min_mtu, max_mtu);
2733                 return -EINVAL;
2734         }
2735
2736         mutex_lock(&priv->state_lock);
2737
2738         reset = !priv->params.lro_en &&
2739                 (priv->params.rq_wq_type !=
2740                  MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
2741
2742         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2743         if (was_opened && reset)
2744                 mlx5e_close_locked(netdev);
2745
2746         netdev->mtu = new_mtu;
2747         mlx5e_set_dev_port_mtu(netdev);
2748
2749         if (was_opened && reset)
2750                 err = mlx5e_open_locked(netdev);
2751
2752         mutex_unlock(&priv->state_lock);
2753
2754         return err;
2755 }
2756
2757 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2758 {
2759         switch (cmd) {
2760         case SIOCSHWTSTAMP:
2761                 return mlx5e_hwstamp_set(dev, ifr);
2762         case SIOCGHWTSTAMP:
2763                 return mlx5e_hwstamp_get(dev, ifr);
2764         default:
2765                 return -EOPNOTSUPP;
2766         }
2767 }
2768
2769 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2770 {
2771         struct mlx5e_priv *priv = netdev_priv(dev);
2772         struct mlx5_core_dev *mdev = priv->mdev;
2773
2774         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2775 }
2776
2777 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2778 {
2779         struct mlx5e_priv *priv = netdev_priv(dev);
2780         struct mlx5_core_dev *mdev = priv->mdev;
2781
2782         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2783                                            vlan, qos);
2784 }
2785
2786 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2787 {
2788         struct mlx5e_priv *priv = netdev_priv(dev);
2789         struct mlx5_core_dev *mdev = priv->mdev;
2790
2791         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2792 }
2793
2794 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2795 {
2796         struct mlx5e_priv *priv = netdev_priv(dev);
2797         struct mlx5_core_dev *mdev = priv->mdev;
2798
2799         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2800 }
2801 static int mlx5_vport_link2ifla(u8 esw_link)
2802 {
2803         switch (esw_link) {
2804         case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
2805                 return IFLA_VF_LINK_STATE_DISABLE;
2806         case MLX5_ESW_VPORT_ADMIN_STATE_UP:
2807                 return IFLA_VF_LINK_STATE_ENABLE;
2808         }
2809         return IFLA_VF_LINK_STATE_AUTO;
2810 }
2811
2812 static int mlx5_ifla_link2vport(u8 ifla_link)
2813 {
2814         switch (ifla_link) {
2815         case IFLA_VF_LINK_STATE_DISABLE:
2816                 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
2817         case IFLA_VF_LINK_STATE_ENABLE:
2818                 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
2819         }
2820         return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
2821 }
2822
2823 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
2824                                    int link_state)
2825 {
2826         struct mlx5e_priv *priv = netdev_priv(dev);
2827         struct mlx5_core_dev *mdev = priv->mdev;
2828
2829         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
2830                                             mlx5_ifla_link2vport(link_state));
2831 }
2832
2833 static int mlx5e_get_vf_config(struct net_device *dev,
2834                                int vf, struct ifla_vf_info *ivi)
2835 {
2836         struct mlx5e_priv *priv = netdev_priv(dev);
2837         struct mlx5_core_dev *mdev = priv->mdev;
2838         int err;
2839
2840         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
2841         if (err)
2842                 return err;
2843         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
2844         return 0;
2845 }
2846
2847 static int mlx5e_get_vf_stats(struct net_device *dev,
2848                               int vf, struct ifla_vf_stats *vf_stats)
2849 {
2850         struct mlx5e_priv *priv = netdev_priv(dev);
2851         struct mlx5_core_dev *mdev = priv->mdev;
2852
2853         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
2854                                             vf_stats);
2855 }
2856
2857 static void mlx5e_add_vxlan_port(struct net_device *netdev,
2858                                  struct udp_tunnel_info *ti)
2859 {
2860         struct mlx5e_priv *priv = netdev_priv(netdev);
2861
2862         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2863                 return;
2864
2865         if (!mlx5e_vxlan_allowed(priv->mdev))
2866                 return;
2867
2868         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
2869 }
2870
2871 static void mlx5e_del_vxlan_port(struct net_device *netdev,
2872                                  struct udp_tunnel_info *ti)
2873 {
2874         struct mlx5e_priv *priv = netdev_priv(netdev);
2875
2876         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2877                 return;
2878
2879         if (!mlx5e_vxlan_allowed(priv->mdev))
2880                 return;
2881
2882         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
2883 }
2884
2885 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
2886                                                     struct sk_buff *skb,
2887                                                     netdev_features_t features)
2888 {
2889         struct udphdr *udph;
2890         u16 proto;
2891         u16 port = 0;
2892
2893         switch (vlan_get_protocol(skb)) {
2894         case htons(ETH_P_IP):
2895                 proto = ip_hdr(skb)->protocol;
2896                 break;
2897         case htons(ETH_P_IPV6):
2898                 proto = ipv6_hdr(skb)->nexthdr;
2899                 break;
2900         default:
2901                 goto out;
2902         }
2903
2904         if (proto == IPPROTO_UDP) {
2905                 udph = udp_hdr(skb);
2906                 port = be16_to_cpu(udph->dest);
2907         }
2908
2909         /* Verify if UDP port is being offloaded by HW */
2910         if (port && mlx5e_vxlan_lookup_port(priv, port))
2911                 return features;
2912
2913 out:
2914         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
2915         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2916 }
2917
2918 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
2919                                               struct net_device *netdev,
2920                                               netdev_features_t features)
2921 {
2922         struct mlx5e_priv *priv = netdev_priv(netdev);
2923
2924         features = vlan_features_check(skb, features);
2925         features = vxlan_features_check(skb, features);
2926
2927         /* Validate if the tunneled packet is being offloaded by HW */
2928         if (skb->encapsulation &&
2929             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
2930                 return mlx5e_vxlan_features_check(priv, skb, features);
2931
2932         return features;
2933 }
2934
2935 static void mlx5e_tx_timeout(struct net_device *dev)
2936 {
2937         struct mlx5e_priv *priv = netdev_priv(dev);
2938         bool sched_work = false;
2939         int i;
2940
2941         netdev_err(dev, "TX timeout detected\n");
2942
2943         for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
2944                 struct mlx5e_sq *sq = priv->txq_to_sq_map[i];
2945
2946                 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
2947                         continue;
2948                 sched_work = true;
2949                 set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
2950                 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
2951                            i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
2952         }
2953
2954         if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
2955                 schedule_work(&priv->tx_timeout_work);
2956 }
2957
2958 static const struct net_device_ops mlx5e_netdev_ops_basic = {
2959         .ndo_open                = mlx5e_open,
2960         .ndo_stop                = mlx5e_close,
2961         .ndo_start_xmit          = mlx5e_xmit,
2962         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
2963         .ndo_select_queue        = mlx5e_select_queue,
2964         .ndo_get_stats64         = mlx5e_get_stats,
2965         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
2966         .ndo_set_mac_address     = mlx5e_set_mac,
2967         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
2968         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
2969         .ndo_set_features        = mlx5e_set_features,
2970         .ndo_change_mtu          = mlx5e_change_mtu,
2971         .ndo_do_ioctl            = mlx5e_ioctl,
2972         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
2973 #ifdef CONFIG_RFS_ACCEL
2974         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
2975 #endif
2976         .ndo_tx_timeout          = mlx5e_tx_timeout,
2977 };
2978
2979 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
2980         .ndo_open                = mlx5e_open,
2981         .ndo_stop                = mlx5e_close,
2982         .ndo_start_xmit          = mlx5e_xmit,
2983         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
2984         .ndo_select_queue        = mlx5e_select_queue,
2985         .ndo_get_stats64         = mlx5e_get_stats,
2986         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
2987         .ndo_set_mac_address     = mlx5e_set_mac,
2988         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
2989         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
2990         .ndo_set_features        = mlx5e_set_features,
2991         .ndo_change_mtu          = mlx5e_change_mtu,
2992         .ndo_do_ioctl            = mlx5e_ioctl,
2993         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
2994         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
2995         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
2996         .ndo_features_check      = mlx5e_features_check,
2997 #ifdef CONFIG_RFS_ACCEL
2998         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
2999 #endif
3000         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
3001         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
3002         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
3003         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
3004         .ndo_get_vf_config       = mlx5e_get_vf_config,
3005         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
3006         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
3007         .ndo_tx_timeout          = mlx5e_tx_timeout,
3008 };
3009
3010 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3011 {
3012         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3013                 return -ENOTSUPP;
3014         if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
3015             !MLX5_CAP_GEN(mdev, nic_flow_table) ||
3016             !MLX5_CAP_ETH(mdev, csum_cap) ||
3017             !MLX5_CAP_ETH(mdev, max_lso_cap) ||
3018             !MLX5_CAP_ETH(mdev, vlan_cap) ||
3019             !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
3020             MLX5_CAP_FLOWTABLE(mdev,
3021                                flow_table_properties_nic_receive.max_ft_level)
3022                                < 3) {
3023                 mlx5_core_warn(mdev,
3024                                "Not creating net device, some required device capabilities are missing\n");
3025                 return -ENOTSUPP;
3026         }
3027         if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
3028                 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
3029         if (!MLX5_CAP_GEN(mdev, cq_moderation))
3030                 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
3031
3032         return 0;
3033 }
3034
3035 u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3036 {
3037         int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
3038
3039         return bf_buf_size -
3040                sizeof(struct mlx5e_tx_wqe) +
3041                2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3042 }
3043
3044 #ifdef CONFIG_MLX5_CORE_EN_DCB
3045 static void mlx5e_ets_init(struct mlx5e_priv *priv)
3046 {
3047         int i;
3048
3049         priv->params.ets.ets_cap = mlx5_max_tc(priv->mdev) + 1;
3050         for (i = 0; i < priv->params.ets.ets_cap; i++) {
3051                 priv->params.ets.tc_tx_bw[i] = MLX5E_MAX_BW_ALLOC;
3052                 priv->params.ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR;
3053                 priv->params.ets.prio_tc[i] = i;
3054         }
3055
3056         /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
3057         priv->params.ets.prio_tc[0] = 1;
3058         priv->params.ets.prio_tc[1] = 0;
3059 }
3060 #endif
3061
3062 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3063                                    u32 *indirection_rqt, int len,
3064                                    int num_channels)
3065 {
3066         int node = mdev->priv.numa_node;
3067         int node_num_of_cores;
3068         int i;
3069
3070         if (node == -1)
3071                 node = first_online_node;
3072
3073         node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3074
3075         if (node_num_of_cores)
3076                 num_channels = min_t(int, num_channels, node_num_of_cores);
3077
3078         for (i = 0; i < len; i++)
3079                 indirection_rqt[i] = i % num_channels;
3080 }
3081
3082 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3083 {
3084         enum pcie_link_width width;
3085         enum pci_bus_speed speed;
3086         int err = 0;
3087
3088         err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3089         if (err)
3090                 return err;
3091
3092         if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3093                 return -EINVAL;
3094
3095         switch (speed) {
3096         case PCIE_SPEED_2_5GT:
3097                 *pci_bw = 2500 * width;
3098                 break;
3099         case PCIE_SPEED_5_0GT:
3100                 *pci_bw = 5000 * width;
3101                 break;
3102         case PCIE_SPEED_8_0GT:
3103                 *pci_bw = 8000 * width;
3104                 break;
3105         default:
3106                 return -EINVAL;
3107         }
3108
3109         return 0;
3110 }
3111
3112 static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3113 {
3114         return (link_speed && pci_bw &&
3115                 (pci_bw < 40000) && (pci_bw < link_speed));
3116 }
3117
3118 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3119 {
3120         params->rx_cq_period_mode = cq_period_mode;
3121
3122         params->rx_cq_moderation.pkts =
3123                 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3124         params->rx_cq_moderation.usec =
3125                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3126
3127         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3128                 params->rx_cq_moderation.usec =
3129                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
3130 }
3131
3132 static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev,
3133                                    u8 *min_inline_mode)
3134 {
3135         switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
3136         case MLX5E_INLINE_MODE_L2:
3137                 *min_inline_mode = MLX5_INLINE_MODE_L2;
3138                 break;
3139         case MLX5E_INLINE_MODE_VPORT_CONTEXT:
3140                 mlx5_query_nic_vport_min_inline(mdev,
3141                                                 min_inline_mode);
3142                 break;
3143         case MLX5_INLINE_MODE_NOT_REQUIRED:
3144                 *min_inline_mode = MLX5_INLINE_MODE_NONE;
3145                 break;
3146         }
3147 }
3148
3149 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3150                                         struct net_device *netdev,
3151                                         const struct mlx5e_profile *profile,
3152                                         void *ppriv)
3153 {
3154         struct mlx5e_priv *priv = netdev_priv(netdev);
3155         u32 link_speed = 0;
3156         u32 pci_bw = 0;
3157         u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3158                                          MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3159                                          MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
3160
3161         priv->mdev                         = mdev;
3162         priv->netdev                       = netdev;
3163         priv->params.num_channels          = profile->max_nch(mdev);
3164         priv->profile                      = profile;
3165         priv->ppriv                        = ppriv;
3166
3167         priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3168
3169         /* set CQE compression */
3170         priv->params.rx_cqe_compress_admin = false;
3171         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
3172             MLX5_CAP_GEN(mdev, vport_group_manager)) {
3173                 mlx5e_get_max_linkspeed(mdev, &link_speed);
3174                 mlx5e_get_pci_bw(mdev, &pci_bw);
3175                 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3176                               link_speed, pci_bw);
3177                 priv->params.rx_cqe_compress_admin =
3178                         cqe_compress_heuristic(link_speed, pci_bw);
3179         }
3180         priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
3181
3182         mlx5e_set_rq_priv_params(priv);
3183         if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
3184                 priv->params.lro_en = true;
3185
3186         priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3187         mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
3188
3189         priv->params.tx_cq_moderation.usec =
3190                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3191         priv->params.tx_cq_moderation.pkts =
3192                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3193         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
3194         mlx5e_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3195         priv->params.num_tc                = 1;
3196         priv->params.rss_hfunc             = ETH_RSS_HASH_XOR;
3197
3198         netdev_rss_key_fill(priv->params.toeplitz_hash_key,
3199                             sizeof(priv->params.toeplitz_hash_key));
3200
3201         mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
3202                                       MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
3203
3204         priv->params.lro_wqe_sz =
3205                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ -
3206                 /* Extra room needed for build_skb */
3207                 MLX5_RX_HEADROOM -
3208                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3209
3210         /* Initialize pflags */
3211         MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3212                             priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
3213
3214 #ifdef CONFIG_MLX5_CORE_EN_DCB
3215         mlx5e_ets_init(priv);
3216 #endif
3217
3218         mutex_init(&priv->state_lock);
3219
3220         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3221         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3222         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
3223         INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3224 }
3225
3226 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3227 {
3228         struct mlx5e_priv *priv = netdev_priv(netdev);
3229
3230         mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
3231         if (is_zero_ether_addr(netdev->dev_addr) &&
3232             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3233                 eth_hw_addr_random(netdev);
3234                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3235         }
3236 }
3237
3238 static const struct switchdev_ops mlx5e_switchdev_ops = {
3239         .switchdev_port_attr_get        = mlx5e_attr_get,
3240 };
3241
3242 static void mlx5e_build_nic_netdev(struct net_device *netdev)
3243 {
3244         struct mlx5e_priv *priv = netdev_priv(netdev);
3245         struct mlx5_core_dev *mdev = priv->mdev;
3246         bool fcs_supported;
3247         bool fcs_enabled;
3248
3249         SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
3250
3251         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3252                 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
3253 #ifdef CONFIG_MLX5_CORE_EN_DCB
3254                 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
3255 #endif
3256         } else {
3257                 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
3258         }
3259
3260         netdev->watchdog_timeo    = 15 * HZ;
3261
3262         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
3263
3264         netdev->vlan_features    |= NETIF_F_SG;
3265         netdev->vlan_features    |= NETIF_F_IP_CSUM;
3266         netdev->vlan_features    |= NETIF_F_IPV6_CSUM;
3267         netdev->vlan_features    |= NETIF_F_GRO;
3268         netdev->vlan_features    |= NETIF_F_TSO;
3269         netdev->vlan_features    |= NETIF_F_TSO6;
3270         netdev->vlan_features    |= NETIF_F_RXCSUM;
3271         netdev->vlan_features    |= NETIF_F_RXHASH;
3272
3273         if (!!MLX5_CAP_ETH(mdev, lro_cap))
3274                 netdev->vlan_features    |= NETIF_F_LRO;
3275
3276         netdev->hw_features       = netdev->vlan_features;
3277         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
3278         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
3279         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
3280
3281         if (mlx5e_vxlan_allowed(mdev)) {
3282                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
3283                                            NETIF_F_GSO_UDP_TUNNEL_CSUM |
3284                                            NETIF_F_GSO_PARTIAL;
3285                 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
3286                 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
3287                 netdev->hw_enc_features |= NETIF_F_TSO;
3288                 netdev->hw_enc_features |= NETIF_F_TSO6;
3289                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
3290                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3291                                            NETIF_F_GSO_PARTIAL;
3292                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
3293         }
3294
3295         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
3296
3297         if (fcs_supported)
3298                 netdev->hw_features |= NETIF_F_RXALL;
3299
3300         netdev->features          = netdev->hw_features;
3301         if (!priv->params.lro_en)
3302                 netdev->features  &= ~NETIF_F_LRO;
3303
3304         if (fcs_enabled)
3305                 netdev->features  &= ~NETIF_F_RXALL;
3306
3307 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3308         if (FT_CAP(flow_modify_en) &&
3309             FT_CAP(modify_root) &&
3310             FT_CAP(identified_miss_table_mode) &&
3311             FT_CAP(flow_table_modify)) {
3312                 netdev->hw_features      |= NETIF_F_HW_TC;
3313 #ifdef CONFIG_RFS_ACCEL
3314                 netdev->hw_features      |= NETIF_F_NTUPLE;
3315 #endif
3316         }
3317
3318         netdev->features         |= NETIF_F_HIGHDMA;
3319
3320         netdev->priv_flags       |= IFF_UNICAST_FLT;
3321
3322         mlx5e_set_netdev_dev_addr(netdev);
3323
3324 #ifdef CONFIG_NET_SWITCHDEV
3325         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3326                 netdev->switchdev_ops = &mlx5e_switchdev_ops;
3327 #endif
3328 }
3329
3330 static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
3331 {
3332         struct mlx5_core_dev *mdev = priv->mdev;
3333         int err;
3334
3335         err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
3336         if (err) {
3337                 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
3338                 priv->q_counter = 0;
3339         }
3340 }
3341
3342 static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
3343 {
3344         if (!priv->q_counter)
3345                 return;
3346
3347         mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
3348 }
3349
3350 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
3351 {
3352         struct mlx5_core_dev *mdev = priv->mdev;
3353         u64 npages = MLX5E_REQUIRED_MTTS(priv->profile->max_nch(mdev),
3354                                          BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW));
3355         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3356         void *mkc;
3357         u32 *in;
3358         int err;
3359
3360         in = mlx5_vzalloc(inlen);
3361         if (!in)
3362                 return -ENOMEM;
3363
3364         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3365
3366         npages = min_t(u32, ALIGN(U16_MAX, 4) * 2, npages);
3367
3368         MLX5_SET(mkc, mkc, free, 1);
3369         MLX5_SET(mkc, mkc, umr_en, 1);
3370         MLX5_SET(mkc, mkc, lw, 1);
3371         MLX5_SET(mkc, mkc, lr, 1);
3372         MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
3373
3374         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3375         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
3376         MLX5_SET64(mkc, mkc, len, npages << PAGE_SHIFT);
3377         MLX5_SET(mkc, mkc, translations_octword_size,
3378                  MLX5_MTT_OCTW(npages));
3379         MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
3380
3381         err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen);
3382
3383         kvfree(in);
3384         return err;
3385 }
3386
3387 static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
3388                            struct net_device *netdev,
3389                            const struct mlx5e_profile *profile,
3390                            void *ppriv)
3391 {
3392         struct mlx5e_priv *priv = netdev_priv(netdev);
3393
3394         mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
3395         mlx5e_build_nic_netdev(netdev);
3396         mlx5e_vxlan_init(priv);
3397 }
3398
3399 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
3400 {
3401         struct mlx5_core_dev *mdev = priv->mdev;
3402         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3403
3404         mlx5e_vxlan_cleanup(priv);
3405
3406         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3407                 mlx5_eswitch_unregister_vport_rep(esw, 0);
3408 }
3409
3410 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
3411 {
3412         struct mlx5_core_dev *mdev = priv->mdev;
3413         int err;
3414         int i;
3415
3416         err = mlx5e_create_indirect_rqts(priv);
3417         if (err) {
3418                 mlx5_core_warn(mdev, "create indirect rqts failed, %d\n", err);
3419                 return err;
3420         }
3421
3422         err = mlx5e_create_direct_rqts(priv);
3423         if (err) {
3424                 mlx5_core_warn(mdev, "create direct rqts failed, %d\n", err);
3425                 goto err_destroy_indirect_rqts;
3426         }
3427
3428         err = mlx5e_create_indirect_tirs(priv);
3429         if (err) {
3430                 mlx5_core_warn(mdev, "create indirect tirs failed, %d\n", err);
3431                 goto err_destroy_direct_rqts;
3432         }
3433
3434         err = mlx5e_create_direct_tirs(priv);
3435         if (err) {
3436                 mlx5_core_warn(mdev, "create direct tirs failed, %d\n", err);
3437                 goto err_destroy_indirect_tirs;
3438         }
3439
3440         err = mlx5e_create_flow_steering(priv);
3441         if (err) {
3442                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
3443                 goto err_destroy_direct_tirs;
3444         }
3445
3446         err = mlx5e_tc_init(priv);
3447         if (err)
3448                 goto err_destroy_flow_steering;
3449
3450         return 0;
3451
3452 err_destroy_flow_steering:
3453         mlx5e_destroy_flow_steering(priv);
3454 err_destroy_direct_tirs:
3455         mlx5e_destroy_direct_tirs(priv);
3456 err_destroy_indirect_tirs:
3457         mlx5e_destroy_indirect_tirs(priv);
3458 err_destroy_direct_rqts:
3459         for (i = 0; i < priv->profile->max_nch(mdev); i++)
3460                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3461 err_destroy_indirect_rqts:
3462         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3463         return err;
3464 }
3465
3466 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
3467 {
3468         int i;
3469
3470         mlx5e_tc_cleanup(priv);
3471         mlx5e_destroy_flow_steering(priv);
3472         mlx5e_destroy_direct_tirs(priv);
3473         mlx5e_destroy_indirect_tirs(priv);
3474         for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
3475                 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
3476         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
3477 }
3478
3479 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
3480 {
3481         int err;
3482
3483         err = mlx5e_create_tises(priv);
3484         if (err) {
3485                 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
3486                 return err;
3487         }
3488
3489 #ifdef CONFIG_MLX5_CORE_EN_DCB
3490         mlx5e_dcbnl_ieee_setets_core(priv, &priv->params.ets);
3491 #endif
3492         return 0;
3493 }
3494
3495 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
3496 {
3497         struct net_device *netdev = priv->netdev;
3498         struct mlx5_core_dev *mdev = priv->mdev;
3499         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3500         struct mlx5_eswitch_rep rep;
3501
3502         mlx5_lag_add(mdev, netdev);
3503
3504         if (mlx5e_vxlan_allowed(mdev)) {
3505                 rtnl_lock();
3506                 udp_tunnel_get_rx_info(netdev);
3507                 rtnl_unlock();
3508         }
3509
3510         mlx5e_enable_async_events(priv);
3511         queue_work(priv->wq, &priv->set_rx_mode_work);
3512
3513         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
3514                 mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
3515                 rep.load = mlx5e_nic_rep_load;
3516                 rep.unload = mlx5e_nic_rep_unload;
3517                 rep.vport = 0;
3518                 rep.priv_data = priv;
3519                 mlx5_eswitch_register_vport_rep(esw, &rep);
3520         }
3521 }
3522
3523 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
3524 {
3525         queue_work(priv->wq, &priv->set_rx_mode_work);
3526         mlx5e_disable_async_events(priv);
3527         mlx5_lag_remove(priv->mdev);
3528 }
3529
3530 static const struct mlx5e_profile mlx5e_nic_profile = {
3531         .init              = mlx5e_nic_init,
3532         .cleanup           = mlx5e_nic_cleanup,
3533         .init_rx           = mlx5e_init_nic_rx,
3534         .cleanup_rx        = mlx5e_cleanup_nic_rx,
3535         .init_tx           = mlx5e_init_nic_tx,
3536         .cleanup_tx        = mlx5e_cleanup_nic_tx,
3537         .enable            = mlx5e_nic_enable,
3538         .disable           = mlx5e_nic_disable,
3539         .update_stats      = mlx5e_update_stats,
3540         .max_nch           = mlx5e_get_max_num_channels,
3541         .max_tc            = MLX5E_MAX_NUM_TC,
3542 };
3543
3544 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
3545                                        const struct mlx5e_profile *profile,
3546                                        void *ppriv)
3547 {
3548         int nch = profile->max_nch(mdev);
3549         struct net_device *netdev;
3550         struct mlx5e_priv *priv;
3551
3552         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
3553                                     nch * profile->max_tc,
3554                                     nch);
3555         if (!netdev) {
3556                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
3557                 return NULL;
3558         }
3559
3560         profile->init(mdev, netdev, profile, ppriv);
3561
3562         netif_carrier_off(netdev);
3563
3564         priv = netdev_priv(netdev);
3565
3566         priv->wq = create_singlethread_workqueue("mlx5e");
3567         if (!priv->wq)
3568                 goto err_cleanup_nic;
3569
3570         return netdev;
3571
3572 err_cleanup_nic:
3573         profile->cleanup(priv);
3574         free_netdev(netdev);
3575
3576         return NULL;
3577 }
3578
3579 int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3580 {
3581         const struct mlx5e_profile *profile;
3582         struct mlx5e_priv *priv;
3583         int err;
3584
3585         priv = netdev_priv(netdev);
3586         profile = priv->profile;
3587         clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
3588
3589         err = mlx5e_create_umr_mkey(priv);
3590         if (err) {
3591                 mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
3592                 goto out;
3593         }
3594
3595         err = profile->init_tx(priv);
3596         if (err)
3597                 goto err_destroy_umr_mkey;
3598
3599         err = mlx5e_open_drop_rq(priv);
3600         if (err) {
3601                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
3602                 goto err_cleanup_tx;
3603         }
3604
3605         err = profile->init_rx(priv);
3606         if (err)
3607                 goto err_close_drop_rq;
3608
3609         mlx5e_create_q_counter(priv);
3610
3611         mlx5e_init_l2_addr(priv);
3612
3613         mlx5e_set_dev_port_mtu(netdev);
3614
3615         if (profile->enable)
3616                 profile->enable(priv);
3617
3618         rtnl_lock();
3619         if (netif_running(netdev))
3620                 mlx5e_open(netdev);
3621         netif_device_attach(netdev);
3622         rtnl_unlock();
3623
3624         return 0;
3625
3626 err_close_drop_rq:
3627         mlx5e_close_drop_rq(priv);
3628
3629 err_cleanup_tx:
3630         profile->cleanup_tx(priv);
3631
3632 err_destroy_umr_mkey:
3633         mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
3634
3635 out:
3636         return err;
3637 }
3638
3639 static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
3640 {
3641         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3642         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3643         int vport;
3644         u8 mac[ETH_ALEN];
3645
3646         if (!MLX5_CAP_GEN(mdev, vport_group_manager))
3647                 return;
3648
3649         mlx5_query_nic_vport_mac_address(mdev, 0, mac);
3650
3651         for (vport = 1; vport < total_vfs; vport++) {
3652                 struct mlx5_eswitch_rep rep;
3653
3654                 rep.load = mlx5e_vport_rep_load;
3655                 rep.unload = mlx5e_vport_rep_unload;
3656                 rep.vport = vport;
3657                 ether_addr_copy(rep.hw_id, mac);
3658                 mlx5_eswitch_register_vport_rep(esw, &rep);
3659         }
3660 }
3661
3662 void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
3663 {
3664         struct mlx5e_priv *priv = netdev_priv(netdev);
3665         const struct mlx5e_profile *profile = priv->profile;
3666
3667         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
3668         if (profile->disable)
3669                 profile->disable(priv);
3670
3671         flush_workqueue(priv->wq);
3672
3673         rtnl_lock();
3674         if (netif_running(netdev))
3675                 mlx5e_close(netdev);
3676         netif_device_detach(netdev);
3677         rtnl_unlock();
3678
3679         mlx5e_destroy_q_counter(priv);
3680         profile->cleanup_rx(priv);
3681         mlx5e_close_drop_rq(priv);
3682         profile->cleanup_tx(priv);
3683         mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
3684         cancel_delayed_work_sync(&priv->update_stats_work);
3685 }
3686
3687 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
3688  * hardware contexts and to connect it to the current netdev.
3689  */
3690 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
3691 {
3692         struct mlx5e_priv *priv = vpriv;
3693         struct net_device *netdev = priv->netdev;
3694         int err;
3695
3696         if (netif_device_present(netdev))
3697                 return 0;
3698
3699         err = mlx5e_create_mdev_resources(mdev);
3700         if (err)
3701                 return err;
3702
3703         err = mlx5e_attach_netdev(mdev, netdev);
3704         if (err) {
3705                 mlx5e_destroy_mdev_resources(mdev);
3706                 return err;
3707         }
3708
3709         return 0;
3710 }
3711
3712 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
3713 {
3714         struct mlx5e_priv *priv = vpriv;
3715         struct net_device *netdev = priv->netdev;
3716
3717         if (!netif_device_present(netdev))
3718                 return;
3719
3720         mlx5e_detach_netdev(mdev, netdev);
3721         mlx5e_destroy_mdev_resources(mdev);
3722 }
3723
3724 static void *mlx5e_add(struct mlx5_core_dev *mdev)
3725 {
3726         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3727         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3728         void *ppriv = NULL;
3729         void *priv;
3730         int vport;
3731         int err;
3732         struct net_device *netdev;
3733
3734         err = mlx5e_check_required_hca_cap(mdev);
3735         if (err)
3736                 return NULL;
3737
3738         mlx5e_register_vport_rep(mdev);
3739
3740         if (MLX5_CAP_GEN(mdev, vport_group_manager))
3741                 ppriv = &esw->offloads.vport_reps[0];
3742
3743         netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
3744         if (!netdev) {
3745                 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
3746                 goto err_unregister_reps;
3747         }
3748
3749         priv = netdev_priv(netdev);
3750
3751         err = mlx5e_attach(mdev, priv);
3752         if (err) {
3753                 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
3754                 goto err_destroy_netdev;
3755         }
3756
3757         err = register_netdev(netdev);
3758         if (err) {
3759                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
3760                 goto err_detach;
3761         }
3762
3763         return priv;
3764
3765 err_detach:
3766         mlx5e_detach(mdev, priv);
3767
3768 err_destroy_netdev:
3769         mlx5e_destroy_netdev(mdev, priv);
3770
3771 err_unregister_reps:
3772         for (vport = 1; vport < total_vfs; vport++)
3773                 mlx5_eswitch_unregister_vport_rep(esw, vport);
3774
3775         return NULL;
3776 }
3777
3778 void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
3779 {
3780         const struct mlx5e_profile *profile = priv->profile;
3781         struct net_device *netdev = priv->netdev;
3782
3783         unregister_netdev(netdev);
3784         destroy_workqueue(priv->wq);
3785         if (profile->cleanup)
3786                 profile->cleanup(priv);
3787         free_netdev(netdev);
3788 }
3789
3790 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
3791 {
3792         struct mlx5_eswitch *esw = mdev->priv.eswitch;
3793         int total_vfs = MLX5_TOTAL_VPORTS(mdev);
3794         struct mlx5e_priv *priv = vpriv;
3795         int vport;
3796
3797         for (vport = 1; vport < total_vfs; vport++)
3798                 mlx5_eswitch_unregister_vport_rep(esw, vport);
3799
3800         mlx5e_detach(mdev, vpriv);
3801         mlx5e_destroy_netdev(mdev, priv);
3802 }
3803
3804 static void *mlx5e_get_netdev(void *vpriv)
3805 {
3806         struct mlx5e_priv *priv = vpriv;
3807
3808         return priv->netdev;
3809 }
3810
3811 static struct mlx5_interface mlx5e_interface = {
3812         .add       = mlx5e_add,
3813         .remove    = mlx5e_remove,
3814         .attach    = mlx5e_attach,
3815         .detach    = mlx5e_detach,
3816         .event     = mlx5e_async_event,
3817         .protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
3818         .get_dev   = mlx5e_get_netdev,
3819 };
3820
3821 void mlx5e_init(void)
3822 {
3823         mlx5e_build_ptys2ethtool_map();
3824         mlx5_register_interface(&mlx5e_interface);
3825 }
3826
3827 void mlx5e_cleanup(void)
3828 {
3829         mlx5_unregister_interface(&mlx5e_interface);
3830 }