drivers/net/ethernet/mellanox/mlx5/core/en_main.c

   1 /*
   2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/tc_act/tc_gact.h>
  34 #include <net/pkt_cls.h>
  35 #include <linux/mlx5/fs.h>
  36 #include <net/vxlan.h>
  37 #include "en.h"
  38 #include "en_tc.h"
  39 #include "eswitch.h"
  40 #include "vxlan.h"
  41
  42 struct mlx5e_rq_param {
  43         u32                        rqc[MLX5_ST_SZ_DW(rqc)];
  44         struct mlx5_wq_param       wq;
  45 };
  46
  47 struct mlx5e_sq_param {
  48         u32                        sqc[MLX5_ST_SZ_DW(sqc)];
  49         struct mlx5_wq_param       wq;
  50         u16                        max_inline;
  51         bool                       icosq;
  52 };
  53
  54 struct mlx5e_cq_param {
  55         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
  56         struct mlx5_wq_param       wq;
  57         u16                        eq_ix;
  58         u8                         cq_period_mode;
  59 };
  60
  61 struct mlx5e_channel_param {
  62         struct mlx5e_rq_param      rq;
  63         struct mlx5e_sq_param      sq;
  64         struct mlx5e_sq_param      icosq;
  65         struct mlx5e_cq_param      rx_cq;
  66         struct mlx5e_cq_param      tx_cq;
  67         struct mlx5e_cq_param      icosq_cq;
  68 };
  69
  70 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
  71 {
  72         struct mlx5_core_dev *mdev = priv->mdev;
  73         u8 port_state;
  74
  75         port_state = mlx5_query_vport_state(mdev,
  76                 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
  77
  78         if (port_state == VPORT_STATE_UP)
  79                 netif_carrier_on(priv->netdev);
  80         else
  81                 netif_carrier_off(priv->netdev);
  82 }
  83
  84 static void mlx5e_update_carrier_work(struct work_struct *work)
  85 {
  86         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
  87                                                update_carrier_work);
  88
  89         mutex_lock(&priv->state_lock);
  90         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
  91                 mlx5e_update_carrier(priv);
  92         mutex_unlock(&priv->state_lock);
  93 }
  94
  95 static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
  96 {
  97         struct mlx5e_sw_stats *s = &priv->stats.sw;
  98         struct mlx5e_rq_stats *rq_stats;
  99         struct mlx5e_sq_stats *sq_stats;
 100         u64 tx_offload_none = 0;
 101         int i, j;
 102
 103         memset(s, 0, sizeof(*s));
 104         for (i = 0; i < priv->params.num_channels; i++) {
 105                 rq_stats = &priv->channel[i]->rq.stats;
 106
 107                 s->rx_packets   += rq_stats->packets;
 108                 s->rx_bytes     += rq_stats->bytes;
 109                 s->lro_packets  += rq_stats->lro_packets;
 110                 s->lro_bytes    += rq_stats->lro_bytes;
 111                 s->rx_csum_none += rq_stats->csum_none;
 112                 s->rx_csum_sw   += rq_stats->csum_sw;
 113                 s->rx_csum_inner += rq_stats->csum_inner;
 114                 s->rx_wqe_err   += rq_stats->wqe_err;
 115                 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
 116                 s->rx_mpwqe_frag   += rq_stats->mpwqe_frag;
 117                 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
 118                 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
 119                 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
 120
 121                 for (j = 0; j < priv->params.num_tc; j++) {
 122                         sq_stats = &priv->channel[i]->sq[j].stats;
 123
 124                         s->tx_packets           += sq_stats->packets;
 125                         s->tx_bytes             += sq_stats->bytes;
 126                         s->tso_packets          += sq_stats->tso_packets;
 127                         s->tso_bytes            += sq_stats->tso_bytes;
 128                         s->tso_inner_packets    += sq_stats->tso_inner_packets;
 129                         s->tso_inner_bytes      += sq_stats->tso_inner_bytes;
 130                         s->tx_queue_stopped     += sq_stats->stopped;
 131                         s->tx_queue_wake        += sq_stats->wake;
 132                         s->tx_queue_dropped     += sq_stats->dropped;
 133                         s->tx_csum_inner        += sq_stats->csum_offload_inner;
 134                         tx_offload_none         += sq_stats->csum_offload_none;
 135                 }
 136         }
 137
 138         /* Update calculated offload counters */
 139         s->tx_csum_offload = s->tx_packets - tx_offload_none - s->tx_csum_inner;
 140         s->rx_csum_good    = s->rx_packets - s->rx_csum_none -
 141                              s->rx_csum_sw;
 142
 143         s->link_down_events = MLX5_GET(ppcnt_reg,
 144                                 priv->stats.pport.phy_counters,
 145                                 counter_set.phys_layer_cntrs.link_down_events);
 146 }
 147
 148 static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
 149 {
 150         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 151         u32 *out = (u32 *)priv->stats.vport.query_vport_out;
 152         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
 153         struct mlx5_core_dev *mdev = priv->mdev;
 154
 155         memset(in, 0, sizeof(in));
 156
 157         MLX5_SET(query_vport_counter_in, in, opcode,
 158                  MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 159         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 160         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 161
 162         memset(out, 0, outlen);
 163
 164         mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 165 }
 166
 167 static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 168 {
 169         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
 170         struct mlx5_core_dev *mdev = priv->mdev;
 171         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 172         int prio;
 173         void *out;
 174         u32 *in;
 175
 176         in = mlx5_vzalloc(sz);
 177         if (!in)
 178                 goto free_out;
 179
 180         MLX5_SET(ppcnt_reg, in, local_port, 1);
 181
 182         out = pstats->IEEE_802_3_counters;
 183         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 184         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 185
 186         out = pstats->RFC_2863_counters;
 187         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 188         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 189
 190         out = pstats->RFC_2819_counters;
 191         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 192         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 193
 194         out = pstats->phy_counters;
 195         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 196         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 197
 198         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 199         for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 200                 out = pstats->per_prio_counters[prio];
 201                 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
 202                 mlx5_core_access_reg(mdev, in, sz, out, sz,
 203                                      MLX5_REG_PPCNT, 0, 0);
 204         }
 205
 206 free_out:
 207         kvfree(in);
 208 }
 209
 210 static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
 211 {
 212         struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
 213
 214         if (!priv->q_counter)
 215                 return;
 216
 217         mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
 218                                       &qcnt->rx_out_of_buffer);
 219 }
 220
 221 void mlx5e_update_stats(struct mlx5e_priv *priv)
 222 {
 223         mlx5e_update_q_counter(priv);
 224         mlx5e_update_vport_counters(priv);
 225         mlx5e_update_pport_counters(priv);
 226         mlx5e_update_sw_counters(priv);
 227 }
 228
 229 static void mlx5e_update_stats_work(struct work_struct *work)
 230 {
 231         struct delayed_work *dwork = to_delayed_work(work);
 232         struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
 233                                                update_stats_work);
 234         mutex_lock(&priv->state_lock);
 235         if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 236                 mlx5e_update_stats(priv);
 237                 queue_delayed_work(priv->wq, dwork,
 238                                    msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
 239         }
 240         mutex_unlock(&priv->state_lock);
 241 }
 242
 243 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
 244                               enum mlx5_dev_event event, unsigned long param)
 245 {
 246         struct mlx5e_priv *priv = vpriv;
 247
 248         if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
 249                 return;
 250
 251         switch (event) {
 252         case MLX5_DEV_EVENT_PORT_UP:
 253         case MLX5_DEV_EVENT_PORT_DOWN:
 254                 queue_work(priv->wq, &priv->update_carrier_work);
 255                 break;
 256
 257         default:
 258                 break;
 259         }
 260 }
 261
 262 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 263 {
 264         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
 265 }
 266
 267 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 268 {
 269         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
 270         synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
 271 }
 272
 273 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 274 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 275
 276 static int mlx5e_create_rq(struct mlx5e_channel *c,
 277                            struct mlx5e_rq_param *param,
 278                            struct mlx5e_rq *rq)
 279 {
 280         struct mlx5e_priv *priv = c->priv;
 281         struct mlx5_core_dev *mdev = priv->mdev;
 282         void *rqc = param->rqc;
 283         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 284         u32 byte_count;
 285         int wq_sz;
 286         int err;
 287         int i;
 288
 289         param->wq.db_numa_node = cpu_to_node(c->cpu);
 290
 291         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
 292                                 &rq->wq_ctrl);
 293         if (err)
 294                 return err;
 295
 296         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
 297
 298         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 299
 300         switch (priv->params.rq_wq_type) {
 301         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 302                 rq->wqe_info = kzalloc_node(wq_sz * sizeof(*rq->wqe_info),
 303                                             GFP_KERNEL, cpu_to_node(c->cpu));
 304                 if (!rq->wqe_info) {
 305                         err = -ENOMEM;
 306                         goto err_rq_wq_destroy;
 307                 }
 308                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
 309                 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
 310
 311                 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
 312                 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
 313                 rq->wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
 314                 byte_count = rq->wqe_sz;
 315                 break;
 316         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 317                 rq->skb = kzalloc_node(wq_sz * sizeof(*rq->skb), GFP_KERNEL,
 318                                        cpu_to_node(c->cpu));
 319                 if (!rq->skb) {
 320                         err = -ENOMEM;
 321                         goto err_rq_wq_destroy;
 322                 }
 323                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
 324                 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
 325
 326                 rq->wqe_sz = (priv->params.lro_en) ?
 327                                 priv->params.lro_wqe_sz :
 328                                 MLX5E_SW2HW_MTU(priv->netdev->mtu);
 329                 rq->wqe_sz = SKB_DATA_ALIGN(rq->wqe_sz);
 330                 byte_count = rq->wqe_sz;
 331                 byte_count |= MLX5_HW_START_PADDING;
 332         }
 333
 334         for (i = 0; i < wq_sz; i++) {
 335                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 336
 337                 wqe->data.byte_count = cpu_to_be32(byte_count);
 338         }
 339
 340         rq->wq_type = priv->params.rq_wq_type;
 341         rq->pdev    = c->pdev;
 342         rq->netdev  = c->netdev;
 343         rq->tstamp  = &priv->tstamp;
 344         rq->channel = c;
 345         rq->ix      = c->ix;
 346         rq->priv    = c->priv;
 347         rq->mkey_be = c->mkey_be;
 348         rq->umr_mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
 349
 350         return 0;
 351
 352 err_rq_wq_destroy:
 353         mlx5_wq_destroy(&rq->wq_ctrl);
 354
 355         return err;
 356 }
 357
 358 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 359 {
 360         switch (rq->wq_type) {
 361         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 362                 kfree(rq->wqe_info);
 363                 break;
 364         default: /* MLX5_WQ_TYPE_LINKED_LIST */
 365                 kfree(rq->skb);
 366         }
 367
 368         mlx5_wq_destroy(&rq->wq_ctrl);
 369 }
 370
 371 static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 372 {
 373         struct mlx5e_priv *priv = rq->priv;
 374         struct mlx5_core_dev *mdev = priv->mdev;
 375
 376         void *in;
 377         void *rqc;
 378         void *wq;
 379         int inlen;
 380         int err;
 381
 382         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 383                 sizeof(u64) * rq->wq_ctrl.buf.npages;
 384         in = mlx5_vzalloc(inlen);
 385         if (!in)
 386                 return -ENOMEM;
 387
 388         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 389         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 390
 391         memcpy(rqc, param->rqc, sizeof(param->rqc));
 392
 393         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 394         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 395         MLX5_SET(rqc,  rqc, flush_in_error_en,  1);
 396         MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
 397         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 398                                                 MLX5_ADAPTER_PAGE_SHIFT);
 399         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 400
 401         mlx5_fill_page_array(&rq->wq_ctrl.buf,
 402                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 403
 404         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 405
 406         kvfree(in);
 407
 408         return err;
 409 }
 410
 411 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
 412                                  int next_state)
 413 {
 414         struct mlx5e_channel *c = rq->channel;
 415         struct mlx5e_priv *priv = c->priv;
 416         struct mlx5_core_dev *mdev = priv->mdev;
 417
 418         void *in;
 419         void *rqc;
 420         int inlen;
 421         int err;
 422
 423         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 424         in = mlx5_vzalloc(inlen);
 425         if (!in)
 426                 return -ENOMEM;
 427
 428         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 429
 430         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 431         MLX5_SET(rqc, rqc, state, next_state);
 432
 433         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 434
 435         kvfree(in);
 436
 437         return err;
 438 }
 439
 440 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 441 {
 442         struct mlx5e_channel *c = rq->channel;
 443         struct mlx5e_priv *priv = c->priv;
 444         struct mlx5_core_dev *mdev = priv->mdev;
 445
 446         void *in;
 447         void *rqc;
 448         int inlen;
 449         int err;
 450
 451         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 452         in = mlx5_vzalloc(inlen);
 453         if (!in)
 454                 return -ENOMEM;
 455
 456         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 457
 458         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 459         MLX5_SET64(modify_rq_in, in, modify_bitmask, MLX5_RQ_BITMASK_VSD);
 460         MLX5_SET(rqc, rqc, vsd, vsd);
 461         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 462
 463         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
 464
 465         kvfree(in);
 466
 467         return err;
 468 }
 469
 470 static void mlx5e_disable_rq(struct mlx5e_rq *rq)
 471 {
 472         mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
 473 }
 474
 475 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
 476 {
 477         unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
 478         struct mlx5e_channel *c = rq->channel;
 479         struct mlx5e_priv *priv = c->priv;
 480         struct mlx5_wq_ll *wq = &rq->wq;
 481
 482         while (time_before(jiffies, exp_time)) {
 483                 if (wq->cur_sz >= priv->params.min_rx_wqes)
 484                         return 0;
 485
 486                 msleep(20);
 487         }
 488
 489         return -ETIMEDOUT;
 490 }
 491
 492 static int mlx5e_open_rq(struct mlx5e_channel *c,
 493                          struct mlx5e_rq_param *param,
 494                          struct mlx5e_rq *rq)
 495 {
 496         struct mlx5e_sq *sq = &c->icosq;
 497         u16 pi = sq->pc & sq->wq.sz_m1;
 498         int err;
 499
 500         err = mlx5e_create_rq(c, param, rq);
 501         if (err)
 502                 return err;
 503
 504         err = mlx5e_enable_rq(rq, param);
 505         if (err)
 506                 goto err_destroy_rq;
 507
 508         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 509         if (err)
 510                 goto err_disable_rq;
 511
 512         set_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
 513
 514         sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;
 515         sq->ico_wqe_info[pi].num_wqebbs = 1;
 516         mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
 517
 518         return 0;
 519
 520 err_disable_rq:
 521         mlx5e_disable_rq(rq);
 522 err_destroy_rq:
 523         mlx5e_destroy_rq(rq);
 524
 525         return err;
 526 }
 527
 528 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 529 {
 530         clear_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
 531         napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
 532
 533         mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
 534         while (!mlx5_wq_ll_is_empty(&rq->wq))
 535                 msleep(20);
 536
 537         /* avoid destroying rq before mlx5e_poll_rx_cq() is done with it */
 538         napi_synchronize(&rq->channel->napi);
 539
 540         mlx5e_disable_rq(rq);
 541         mlx5e_destroy_rq(rq);
 542 }
 543
 544 static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
 545 {
 546         kfree(sq->wqe_info);
 547         kfree(sq->dma_fifo);
 548         kfree(sq->skb);
 549 }
 550
 551 static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
 552 {
 553         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 554         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 555
 556         sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);
 557         sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,
 558                                     numa);
 559         sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,
 560                                     numa);
 561
 562         if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {
 563                 mlx5e_free_sq_db(sq);
 564                 return -ENOMEM;
 565         }
 566
 567         sq->dma_fifo_mask = df_sz - 1;
 568
 569         return 0;
 570 }
 571
 572 static int mlx5e_create_sq(struct mlx5e_channel *c,
 573                            int tc,
 574                            struct mlx5e_sq_param *param,
 575                            struct mlx5e_sq *sq)
 576 {
 577         struct mlx5e_priv *priv = c->priv;
 578         struct mlx5_core_dev *mdev = priv->mdev;
 579
 580         void *sqc = param->sqc;
 581         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
 582         int err;
 583
 584         err = mlx5_alloc_map_uar(mdev, &sq->uar, true);
 585         if (err)
 586                 return err;
 587
 588         param->wq.db_numa_node = cpu_to_node(c->cpu);
 589
 590         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
 591                                  &sq->wq_ctrl);
 592         if (err)
 593                 goto err_unmap_free_uar;
 594
 595         sq->wq.db       = &sq->wq.db[MLX5_SND_DBR];
 596         if (sq->uar.bf_map) {
 597                 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
 598                 sq->uar_map = sq->uar.bf_map;
 599         } else {
 600                 sq->uar_map = sq->uar.map;
 601         }
 602         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
 603         sq->max_inline  = param->max_inline;
 604
 605         err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
 606         if (err)
 607                 goto err_sq_wq_destroy;
 608
 609         if (param->icosq) {
 610                 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 611
 612                 sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *
 613                                                 wq_sz,
 614                                                 GFP_KERNEL,
 615                                                 cpu_to_node(c->cpu));
 616                 if (!sq->ico_wqe_info) {
 617                         err = -ENOMEM;
 618                         goto err_free_sq_db;
 619                 }
 620         } else {
 621                 int txq_ix;
 622
 623                 txq_ix = c->ix + tc * priv->params.num_channels;
 624                 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
 625                 priv->txq_to_sq_map[txq_ix] = sq;
 626         }
 627
 628         sq->pdev      = c->pdev;
 629         sq->tstamp    = &priv->tstamp;
 630         sq->mkey_be   = c->mkey_be;
 631         sq->channel   = c;
 632         sq->tc        = tc;
 633         sq->edge      = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
 634         sq->bf_budget = MLX5E_SQ_BF_BUDGET;
 635
 636         return 0;
 637
 638 err_free_sq_db:
 639         mlx5e_free_sq_db(sq);
 640
 641 err_sq_wq_destroy:
 642         mlx5_wq_destroy(&sq->wq_ctrl);
 643
 644 err_unmap_free_uar:
 645         mlx5_unmap_free_uar(mdev, &sq->uar);
 646
 647         return err;
 648 }
 649
 650 static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
 651 {
 652         struct mlx5e_channel *c = sq->channel;
 653         struct mlx5e_priv *priv = c->priv;
 654
 655         kfree(sq->ico_wqe_info);
 656         mlx5e_free_sq_db(sq);
 657         mlx5_wq_destroy(&sq->wq_ctrl);
 658         mlx5_unmap_free_uar(priv->mdev, &sq->uar);
 659 }
 660
 661 static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
 662 {
 663         struct mlx5e_channel *c = sq->channel;
 664         struct mlx5e_priv *priv = c->priv;
 665         struct mlx5_core_dev *mdev = priv->mdev;
 666
 667         void *in;
 668         void *sqc;
 669         void *wq;
 670         int inlen;
 671         int err;
 672
 673         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
 674                 sizeof(u64) * sq->wq_ctrl.buf.npages;
 675         in = mlx5_vzalloc(inlen);
 676         if (!in)
 677                 return -ENOMEM;
 678
 679         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 680         wq = MLX5_ADDR_OF(sqc, sqc, wq);
 681
 682         memcpy(sqc, param->sqc, sizeof(param->sqc));
 683
 684         MLX5_SET(sqc,  sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);
 685         MLX5_SET(sqc,  sqc, cqn,                sq->cq.mcq.cqn);
 686         MLX5_SET(sqc,  sqc, state,              MLX5_SQC_STATE_RST);
 687         MLX5_SET(sqc,  sqc, tis_lst_sz,         param->icosq ? 0 : 1);
 688         MLX5_SET(sqc,  sqc, flush_in_error_en,  1);
 689
 690         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
 691         MLX5_SET(wq,   wq, uar_page,      sq->uar.index);
 692         MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
 693                                           MLX5_ADAPTER_PAGE_SHIFT);
 694         MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
 695
 696         mlx5_fill_page_array(&sq->wq_ctrl.buf,
 697                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 698
 699         err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
 700
 701         kvfree(in);
 702
 703         return err;
 704 }
 705
 706 static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
 707                            int next_state, bool update_rl, int rl_index)
 708 {
 709         struct mlx5e_channel *c = sq->channel;
 710         struct mlx5e_priv *priv = c->priv;
 711         struct mlx5_core_dev *mdev = priv->mdev;
 712
 713         void *in;
 714         void *sqc;
 715         int inlen;
 716         int err;
 717
 718         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
 719         in = mlx5_vzalloc(inlen);
 720         if (!in)
 721                 return -ENOMEM;
 722
 723         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
 724
 725         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
 726         MLX5_SET(sqc, sqc, state, next_state);
 727         if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
 728                 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
 729                 MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
 730         }
 731
 732         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
 733
 734         kvfree(in);
 735
 736         return err;
 737 }
 738
 739 static void mlx5e_disable_sq(struct mlx5e_sq *sq)
 740 {
 741         struct mlx5e_channel *c = sq->channel;
 742         struct mlx5e_priv *priv = c->priv;
 743         struct mlx5_core_dev *mdev = priv->mdev;
 744
 745         mlx5_core_destroy_sq(mdev, sq->sqn);
 746         if (sq->rate_limit)
 747                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
 748 }
 749
 750 static int mlx5e_open_sq(struct mlx5e_channel *c,
 751                          int tc,
 752                          struct mlx5e_sq_param *param,
 753                          struct mlx5e_sq *sq)
 754 {
 755         int err;
 756
 757         err = mlx5e_create_sq(c, tc, param, sq);
 758         if (err)
 759                 return err;
 760
 761         err = mlx5e_enable_sq(sq, param);
 762         if (err)
 763                 goto err_destroy_sq;
 764
 765         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
 766                               false, 0);
 767         if (err)
 768                 goto err_disable_sq;
 769
 770         if (sq->txq) {
 771                 set_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state);
 772                 netdev_tx_reset_queue(sq->txq);
 773                 netif_tx_start_queue(sq->txq);
 774         }
 775
 776         return 0;
 777
 778 err_disable_sq:
 779         mlx5e_disable_sq(sq);
 780 err_destroy_sq:
 781         mlx5e_destroy_sq(sq);
 782
 783         return err;
 784 }
 785
 786 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
 787 {
 788         __netif_tx_lock_bh(txq);
 789         netif_tx_stop_queue(txq);
 790         __netif_tx_unlock_bh(txq);
 791 }
 792
 793 static void mlx5e_close_sq(struct mlx5e_sq *sq)
 794 {
 795         if (sq->txq) {
 796                 clear_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state);
 797                 /* prevent netif_tx_wake_queue */
 798                 napi_synchronize(&sq->channel->napi);
 799                 netif_tx_disable_queue(sq->txq);
 800
 801                 /* ensure hw is notified of all pending wqes */
 802                 if (mlx5e_sq_has_room_for(sq, 1))
 803                         mlx5e_send_nop(sq, true);
 804
 805                 mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR,
 806                                 false, 0);
 807         }
 808
 809         while (sq->cc != sq->pc) /* wait till sq is empty */
 810                 msleep(20);
 811
 812         /* avoid destroying sq before mlx5e_poll_tx_cq() is done with it */
 813         napi_synchronize(&sq->channel->napi);
 814
 815         mlx5e_disable_sq(sq);
 816         mlx5e_destroy_sq(sq);
 817 }
 818
 819 static int mlx5e_create_cq(struct mlx5e_channel *c,
 820                            struct mlx5e_cq_param *param,
 821                            struct mlx5e_cq *cq)
 822 {
 823         struct mlx5e_priv *priv = c->priv;
 824         struct mlx5_core_dev *mdev = priv->mdev;
 825         struct mlx5_core_cq *mcq = &cq->mcq;
 826         int eqn_not_used;
 827         unsigned int irqn;
 828         int err;
 829         u32 i;
 830
 831         param->wq.buf_numa_node = cpu_to_node(c->cpu);
 832         param->wq.db_numa_node  = cpu_to_node(c->cpu);
 833         param->eq_ix   = c->ix;
 834
 835         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
 836                                &cq->wq_ctrl);
 837         if (err)
 838                 return err;
 839
 840         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
 841
 842         cq->napi        = &c->napi;
 843
 844         mcq->cqe_sz     = 64;
 845         mcq->set_ci_db  = cq->wq_ctrl.db.db;
 846         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
 847         *mcq->set_ci_db = 0;
 848         *mcq->arm_db    = 0;
 849         mcq->vector     = param->eq_ix;
 850         mcq->comp       = mlx5e_completion_event;
 851         mcq->event      = mlx5e_cq_error_event;
 852         mcq->irqn       = irqn;
 853         mcq->uar        = &priv->cq_uar;
 854
 855         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
 856                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
 857
 858                 cqe->op_own = 0xf1;
 859         }
 860
 861         cq->channel = c;
 862         cq->priv = priv;
 863
 864         return 0;
 865 }
 866
 867 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
 868 {
 869         mlx5_wq_destroy(&cq->wq_ctrl);
 870 }
 871
 872 static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
 873 {
 874         struct mlx5e_priv *priv = cq->priv;
 875         struct mlx5_core_dev *mdev = priv->mdev;
 876         struct mlx5_core_cq *mcq = &cq->mcq;
 877
 878         void *in;
 879         void *cqc;
 880         int inlen;
 881         unsigned int irqn_not_used;
 882         int eqn;
 883         int err;
 884
 885         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
 886                 sizeof(u64) * cq->wq_ctrl.buf.npages;
 887         in = mlx5_vzalloc(inlen);
 888         if (!in)
 889                 return -ENOMEM;
 890
 891         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 892
 893         memcpy(cqc, param->cqc, sizeof(param->cqc));
 894
 895         mlx5_fill_page_array(&cq->wq_ctrl.buf,
 896                              (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
 897
 898         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
 899
 900         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
 901         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
 902         MLX5_SET(cqc,   cqc, uar_page,      mcq->uar->index);
 903         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
 904                                             MLX5_ADAPTER_PAGE_SHIFT);
 905         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
 906
 907         err = mlx5_core_create_cq(mdev, mcq, in, inlen);
 908
 909         kvfree(in);
 910
 911         if (err)
 912                 return err;
 913
 914         mlx5e_cq_arm(cq);
 915
 916         return 0;
 917 }
 918
 919 static void mlx5e_disable_cq(struct mlx5e_cq *cq)
 920 {
 921         struct mlx5e_priv *priv = cq->priv;
 922         struct mlx5_core_dev *mdev = priv->mdev;
 923
 924         mlx5_core_destroy_cq(mdev, &cq->mcq);
 925 }
 926
 927 static int mlx5e_open_cq(struct mlx5e_channel *c,
 928                          struct mlx5e_cq_param *param,
 929                          struct mlx5e_cq *cq,
 930                          struct mlx5e_cq_moder moderation)
 931 {
 932         int err;
 933         struct mlx5e_priv *priv = c->priv;
 934         struct mlx5_core_dev *mdev = priv->mdev;
 935
 936         err = mlx5e_create_cq(c, param, cq);
 937         if (err)
 938                 return err;
 939
 940         err = mlx5e_enable_cq(cq, param);
 941         if (err)
 942                 goto err_destroy_cq;
 943
 944         if (MLX5_CAP_GEN(mdev, cq_moderation))
 945                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
 946                                                moderation.usec,
 947                                                moderation.pkts);
 948         return 0;
 949
 950 err_destroy_cq:
 951         mlx5e_destroy_cq(cq);
 952
 953         return err;
 954 }
 955
 956 static void mlx5e_close_cq(struct mlx5e_cq *cq)
 957 {
 958         mlx5e_disable_cq(cq);
 959         mlx5e_destroy_cq(cq);
 960 }
 961
 962 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
 963 {
 964         return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
 965 }
 966
 967 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
 968                              struct mlx5e_channel_param *cparam)
 969 {
 970         struct mlx5e_priv *priv = c->priv;
 971         int err;
 972         int tc;
 973
 974         for (tc = 0; tc < c->num_tc; tc++) {
 975                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
 976                                     priv->params.tx_cq_moderation);
 977                 if (err)
 978                         goto err_close_tx_cqs;
 979         }
 980
 981         return 0;
 982
 983 err_close_tx_cqs:
 984         for (tc--; tc >= 0; tc--)
 985                 mlx5e_close_cq(&c->sq[tc].cq);
 986
 987         return err;
 988 }
 989
 990 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
 991 {
 992         int tc;
 993
 994         for (tc = 0; tc < c->num_tc; tc++)
 995                 mlx5e_close_cq(&c->sq[tc].cq);
 996 }
 997
 998 static int mlx5e_open_sqs(struct mlx5e_channel *c,
 999                           struct mlx5e_channel_param *cparam)
1000 {
1001         int err;
1002         int tc;
1003
1004         for (tc = 0; tc < c->num_tc; tc++) {
1005                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1006                 if (err)
1007                         goto err_close_sqs;
1008         }
1009
1010         return 0;
1011
1012 err_close_sqs:
1013         for (tc--; tc >= 0; tc--)
1014                 mlx5e_close_sq(&c->sq[tc]);
1015
1016         return err;
1017 }
1018
1019 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1020 {
1021         int tc;
1022
1023         for (tc = 0; tc < c->num_tc; tc++)
1024                 mlx5e_close_sq(&c->sq[tc]);
1025 }
1026
1027 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
1028 {
1029         int i;
1030
1031         for (i = 0; i < MLX5E_MAX_NUM_TC; i++)
1032                 priv->channeltc_to_txq_map[ix][i] =
1033                         ix + i * priv->params.num_channels;
1034 }
1035
1036 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1037                                 struct mlx5e_sq *sq, u32 rate)
1038 {
1039         struct mlx5e_priv *priv = netdev_priv(dev);
1040         struct mlx5_core_dev *mdev = priv->mdev;
1041         u16 rl_index = 0;
1042         int err;
1043
1044         if (rate == sq->rate_limit)
1045                 /* nothing to do */
1046                 return 0;
1047
1048         if (sq->rate_limit)
1049                 /* remove current rl index to free space to next ones */
1050                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1051
1052         sq->rate_limit = 0;
1053
1054         if (rate) {
1055                 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1056                 if (err) {
1057                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1058                                    rate, err);
1059                         return err;
1060                 }
1061         }
1062
1063         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1064                               MLX5_SQC_STATE_RDY, true, rl_index);
1065         if (err) {
1066                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1067                            rate, err);
1068                 /* remove the rate from the table */
1069                 if (rate)
1070                         mlx5_rl_remove_rate(mdev, rate);
1071                 return err;
1072         }
1073
1074         sq->rate_limit = rate;
1075         return 0;
1076 }
1077
1078 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1079 {
1080         struct mlx5e_priv *priv = netdev_priv(dev);
1081         struct mlx5_core_dev *mdev = priv->mdev;
1082         struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1083         int err = 0;
1084
1085         if (!mlx5_rl_is_supported(mdev)) {
1086                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1087                 return -EINVAL;
1088         }
1089
1090         /* rate is given in Mb/sec, HW config is in Kb/sec */
1091         rate = rate << 10;
1092
1093         /* Check whether rate in valid range, 0 is always valid */
1094         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1095                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1096                 return -ERANGE;
1097         }
1098
1099         mutex_lock(&priv->state_lock);
1100         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1101                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1102         if (!err)
1103                 priv->tx_rates[index] = rate;
1104         mutex_unlock(&priv->state_lock);
1105
1106         return err;
1107 }
1108
1109 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1110                               struct mlx5e_channel_param *cparam,
1111                               struct mlx5e_channel **cp)
1112 {
1113         struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
1114         struct net_device *netdev = priv->netdev;
1115         int cpu = mlx5e_get_cpu(priv, ix);
1116         struct mlx5e_channel *c;
1117         struct mlx5e_sq *sq;
1118         int err;
1119         int i;
1120
1121         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1122         if (!c)
1123                 return -ENOMEM;
1124
1125         c->priv     = priv;
1126         c->ix       = ix;
1127         c->cpu      = cpu;
1128         c->pdev     = &priv->mdev->pdev->dev;
1129         c->netdev   = priv->netdev;
1130         c->mkey_be  = cpu_to_be32(priv->mkey.key);
1131         c->num_tc   = priv->params.num_tc;
1132
1133         mlx5e_build_channeltc_to_txq_map(priv, ix);
1134
1135         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1136
1137         err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
1138         if (err)
1139                 goto err_napi_del;
1140
1141         err = mlx5e_open_tx_cqs(c, cparam);
1142         if (err)
1143                 goto err_close_icosq_cq;
1144
1145         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1146                             priv->params.rx_cq_moderation);
1147         if (err)
1148                 goto err_close_tx_cqs;
1149
1150         napi_enable(&c->napi);
1151
1152         err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
1153         if (err)
1154                 goto err_disable_napi;
1155
1156         err = mlx5e_open_sqs(c, cparam);
1157         if (err)
1158                 goto err_close_icosq;
1159
1160         for (i = 0; i < priv->params.num_tc; i++) {
1161                 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1162
1163                 if (priv->tx_rates[txq_ix]) {
1164                         sq = priv->txq_to_sq_map[txq_ix];
1165                         mlx5e_set_sq_maxrate(priv->netdev, sq,
1166                                              priv->tx_rates[txq_ix]);
1167                 }
1168         }
1169
1170         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1171         if (err)
1172                 goto err_close_sqs;
1173
1174         netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1175         *cp = c;
1176
1177         return 0;
1178
1179 err_close_sqs:
1180         mlx5e_close_sqs(c);
1181
1182 err_close_icosq:
1183         mlx5e_close_sq(&c->icosq);
1184
1185 err_disable_napi:
1186         napi_disable(&c->napi);
1187         mlx5e_close_cq(&c->rq.cq);
1188
1189 err_close_tx_cqs:
1190         mlx5e_close_tx_cqs(c);
1191
1192 err_close_icosq_cq:
1193         mlx5e_close_cq(&c->icosq.cq);
1194
1195 err_napi_del:
1196         netif_napi_del(&c->napi);
1197         napi_hash_del(&c->napi);
1198         kfree(c);
1199
1200         return err;
1201 }
1202
1203 static void mlx5e_close_channel(struct mlx5e_channel *c)
1204 {
1205         mlx5e_close_rq(&c->rq);
1206         mlx5e_close_sqs(c);
1207         mlx5e_close_sq(&c->icosq);
1208         napi_disable(&c->napi);
1209         mlx5e_close_cq(&c->rq.cq);
1210         mlx5e_close_tx_cqs(c);
1211         mlx5e_close_cq(&c->icosq.cq);
1212         netif_napi_del(&c->napi);
1213
1214         napi_hash_del(&c->napi);
1215         synchronize_rcu();
1216
1217         kfree(c);
1218 }
1219
1220 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1221                                  struct mlx5e_rq_param *param)
1222 {
1223         void *rqc = param->rqc;
1224         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1225
1226         switch (priv->params.rq_wq_type) {
1227         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1228                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1229                          priv->params.mpwqe_log_num_strides - 9);
1230                 MLX5_SET(wq, wq, log_wqe_stride_size,
1231                          priv->params.mpwqe_log_stride_sz - 6);
1232                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1233                 break;
1234         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1235                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1236         }
1237
1238         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1239         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1240         MLX5_SET(wq, wq, log_wq_sz,        priv->params.log_rq_size);
1241         MLX5_SET(wq, wq, pd,               priv->pdn);
1242         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1243
1244         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1245         param->wq.linear = 1;
1246 }
1247
1248 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1249 {
1250         void *rqc = param->rqc;
1251         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1252
1253         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1254         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1255 }
1256
1257 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1258                                         struct mlx5e_sq_param *param)
1259 {
1260         void *sqc = param->sqc;
1261         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1262
1263         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1264         MLX5_SET(wq, wq, pd,            priv->pdn);
1265
1266         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1267 }
1268
1269 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1270                                  struct mlx5e_sq_param *param)
1271 {
1272         void *sqc = param->sqc;
1273         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1274
1275         mlx5e_build_sq_param_common(priv, param);
1276         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1277
1278         param->max_inline = priv->params.tx_max_inline;
1279 }
1280
1281 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1282                                         struct mlx5e_cq_param *param)
1283 {
1284         void *cqc = param->cqc;
1285
1286         MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
1287 }
1288
1289 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1290                                     struct mlx5e_cq_param *param)
1291 {
1292         void *cqc = param->cqc;
1293         u8 log_cq_size;
1294
1295         switch (priv->params.rq_wq_type) {
1296         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1297                 log_cq_size = priv->params.log_rq_size +
1298                         priv->params.mpwqe_log_num_strides;
1299                 break;
1300         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1301                 log_cq_size = priv->params.log_rq_size;
1302         }
1303
1304         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
1305         if (priv->params.rx_cqe_compress) {
1306                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1307                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1308         }
1309
1310         mlx5e_build_common_cq_param(priv, param);
1311
1312         param->cq_period_mode = priv->params.rx_cq_period_mode;
1313 }
1314
1315 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1316                                     struct mlx5e_cq_param *param)
1317 {
1318         void *cqc = param->cqc;
1319
1320         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1321
1322         mlx5e_build_common_cq_param(priv, param);
1323
1324         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1325 }
1326
1327 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1328                                      struct mlx5e_cq_param *param,
1329                                      u8 log_wq_size)
1330 {
1331         void *cqc = param->cqc;
1332
1333         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1334
1335         mlx5e_build_common_cq_param(priv, param);
1336
1337         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1338 }
1339
1340 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1341                                     struct mlx5e_sq_param *param,
1342                                     u8 log_wq_size)
1343 {
1344         void *sqc = param->sqc;
1345         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1346
1347         mlx5e_build_sq_param_common(priv, param);
1348
1349         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1350         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
1351
1352         param->icosq = true;
1353 }
1354
1355 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
1356 {
1357         u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1358
1359         mlx5e_build_rq_param(priv, &cparam->rq);
1360         mlx5e_build_sq_param(priv, &cparam->sq);
1361         mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
1362         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1363         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1364         mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
1365 }
1366
1367 static int mlx5e_open_channels(struct mlx5e_priv *priv)
1368 {
1369         struct mlx5e_channel_param *cparam;
1370         int nch = priv->params.num_channels;
1371         int err = -ENOMEM;
1372         int i;
1373         int j;
1374
1375         priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1376                                 GFP_KERNEL);
1377
1378         priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
1379                                       sizeof(struct mlx5e_sq *), GFP_KERNEL);
1380
1381         cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1382
1383         if (!priv->channel || !priv->txq_to_sq_map || !cparam)
1384                 goto err_free_txq_to_sq_map;
1385
1386         mlx5e_build_channel_param(priv, cparam);
1387
1388         for (i = 0; i < nch; i++) {
1389                 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
1390                 if (err)
1391                         goto err_close_channels;
1392         }
1393
1394         for (j = 0; j < nch; j++) {
1395                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1396                 if (err)
1397                         goto err_close_channels;
1398         }
1399
1400         kfree(cparam);
1401         return 0;
1402
1403 err_close_channels:
1404         for (i--; i >= 0; i--)
1405                 mlx5e_close_channel(priv->channel[i]);
1406
1407 err_free_txq_to_sq_map:
1408         kfree(priv->txq_to_sq_map);
1409         kfree(priv->channel);
1410         kfree(cparam);
1411
1412         return err;
1413 }
1414
1415 static void mlx5e_close_channels(struct mlx5e_priv *priv)
1416 {
1417         int i;
1418
1419         for (i = 0; i < priv->params.num_channels; i++)
1420                 mlx5e_close_channel(priv->channel[i]);
1421
1422         kfree(priv->txq_to_sq_map);
1423         kfree(priv->channel);
1424 }
1425
1426 static int mlx5e_rx_hash_fn(int hfunc)
1427 {
1428         return (hfunc == ETH_RSS_HASH_TOP) ?
1429                MLX5_RX_HASH_FN_TOEPLITZ :
1430                MLX5_RX_HASH_FN_INVERTED_XOR8;
1431 }
1432
1433 static int mlx5e_bits_invert(unsigned long a, int size)
1434 {
1435         int inv = 0;
1436         int i;
1437
1438         for (i = 0; i < size; i++)
1439                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1440
1441         return inv;
1442 }
1443
1444 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1445 {
1446         int i;
1447
1448         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1449                 int ix = i;
1450                 u32 rqn;
1451
1452                 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1453                         ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1454
1455                 ix = priv->params.indirection_rqt[ix];
1456                 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1457                                 priv->channel[ix]->rq.rqn :
1458                                 priv->drop_rq.rqn;
1459                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
1460         }
1461 }
1462
1463 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1464                                       int ix)
1465 {
1466         u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1467                         priv->channel[ix]->rq.rqn :
1468                         priv->drop_rq.rqn;
1469
1470         MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
1471 }
1472
1473 static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, int ix, u32 *rqtn)
1474 {
1475         struct mlx5_core_dev *mdev = priv->mdev;
1476         void *rqtc;
1477         int inlen;
1478         int err;
1479         u32 *in;
1480
1481         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1482         in = mlx5_vzalloc(inlen);
1483         if (!in)
1484                 return -ENOMEM;
1485
1486         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1487
1488         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1489         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1490
1491         if (sz > 1) /* RSS */
1492                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1493         else
1494                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1495
1496         err = mlx5_core_create_rqt(mdev, in, inlen, rqtn);
1497
1498         kvfree(in);
1499         return err;
1500 }
1501
1502 static void mlx5e_destroy_rqt(struct mlx5e_priv *priv, u32 rqtn)
1503 {
1504         mlx5_core_destroy_rqt(priv->mdev, rqtn);
1505 }
1506
1507 static int mlx5e_create_rqts(struct mlx5e_priv *priv)
1508 {
1509         int nch = mlx5e_get_max_num_channels(priv->mdev);
1510         u32 *rqtn;
1511         int err;
1512         int ix;
1513
1514         /* Indirect RQT */
1515         rqtn = &priv->indir_rqtn;
1516         err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqtn);
1517         if (err)
1518                 return err;
1519
1520         /* Direct RQTs */
1521         for (ix = 0; ix < nch; ix++) {
1522                 rqtn = &priv->direct_tir[ix].rqtn;
1523                 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqtn);
1524                 if (err)
1525                         goto err_destroy_rqts;
1526         }
1527
1528         return 0;
1529
1530 err_destroy_rqts:
1531         for (ix--; ix >= 0; ix--)
1532                 mlx5e_destroy_rqt(priv, priv->direct_tir[ix].rqtn);
1533
1534         mlx5e_destroy_rqt(priv, priv->indir_rqtn);
1535
1536         return err;
1537 }
1538
1539 static void mlx5e_destroy_rqts(struct mlx5e_priv *priv)
1540 {
1541         int nch = mlx5e_get_max_num_channels(priv->mdev);
1542         int i;
1543
1544         for (i = 0; i < nch; i++)
1545                 mlx5e_destroy_rqt(priv, priv->direct_tir[i].rqtn);
1546
1547         mlx5e_destroy_rqt(priv, priv->indir_rqtn);
1548 }
1549
1550 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
1551 {
1552         struct mlx5_core_dev *mdev = priv->mdev;
1553         void *rqtc;
1554         int inlen;
1555         u32 *in;
1556         int err;
1557
1558         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1559         in = mlx5_vzalloc(inlen);
1560         if (!in)
1561                 return -ENOMEM;
1562
1563         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1564
1565         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1566         if (sz > 1) /* RSS */
1567                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1568         else
1569                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1570
1571         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1572
1573         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
1574
1575         kvfree(in);
1576
1577         return err;
1578 }
1579
1580 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1581 {
1582         u32 rqtn;
1583         int ix;
1584
1585         rqtn = priv->indir_rqtn;
1586         mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
1587         for (ix = 0; ix < priv->params.num_channels; ix++) {
1588                 rqtn = priv->direct_tir[ix].rqtn;
1589                 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
1590         }
1591 }
1592
1593 static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
1594 {
1595         if (!priv->params.lro_en)
1596                 return;
1597
1598 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1599
1600         MLX5_SET(tirc, tirc, lro_enable_mask,
1601                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1602                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1603         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
1604                  (priv->params.lro_wqe_sz -
1605                   ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1606         MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1607                  MLX5_CAP_ETH(priv->mdev,
1608                               lro_timer_supported_periods[2]));
1609 }
1610
1611 void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
1612 {
1613         MLX5_SET(tirc, tirc, rx_hash_fn,
1614                  mlx5e_rx_hash_fn(priv->params.rss_hfunc));
1615         if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
1616                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
1617                                              rx_hash_toeplitz_key);
1618                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
1619                                                rx_hash_toeplitz_key);
1620
1621                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1622                 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
1623         }
1624 }
1625
1626 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
1627 {
1628         struct mlx5_core_dev *mdev = priv->mdev;
1629
1630         void *in;
1631         void *tirc;
1632         int inlen;
1633         int err;
1634         int tt;
1635         int ix;
1636
1637         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1638         in = mlx5_vzalloc(inlen);
1639         if (!in)
1640                 return -ENOMEM;
1641
1642         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
1643         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
1644
1645         mlx5e_build_tir_ctx_lro(tirc, priv);
1646
1647         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
1648                 err = mlx5_core_modify_tir(mdev, priv->indir_tirn[tt], in,
1649                                            inlen);
1650                 if (err)
1651                         goto free_in;
1652         }
1653
1654         for (ix = 0; ix < mlx5e_get_max_num_channels(mdev); ix++) {
1655                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
1656                                            in, inlen);
1657                 if (err)
1658                         goto free_in;
1659         }
1660
1661 free_in:
1662         kvfree(in);
1663
1664         return err;
1665 }
1666
1667 static int mlx5e_refresh_tirs_self_loopback_enable(struct mlx5e_priv *priv)
1668 {
1669         void *in;
1670         int inlen;
1671         int err;
1672         int i;
1673
1674         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
1675         in = mlx5_vzalloc(inlen);
1676         if (!in)
1677                 return -ENOMEM;
1678
1679         MLX5_SET(modify_tir_in, in, bitmask.self_lb_en, 1);
1680
1681         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
1682                 err = mlx5_core_modify_tir(priv->mdev, priv->indir_tirn[i], in,
1683                                            inlen);
1684                 if (err)
1685                         return err;
1686         }
1687
1688         for (i = 0; i < priv->params.num_channels; i++) {
1689                 err = mlx5_core_modify_tir(priv->mdev,
1690                                            priv->direct_tir[i].tirn, in,
1691                                            inlen);
1692                 if (err)
1693                         return err;
1694         }
1695
1696         kvfree(in);
1697
1698         return 0;
1699 }
1700
1701 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
1702 {
1703         struct mlx5_core_dev *mdev = priv->mdev;
1704         u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
1705         int err;
1706
1707         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
1708         if (err)
1709                 return err;
1710
1711         /* Update vport context MTU */
1712         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
1713         return 0;
1714 }
1715
1716 static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
1717 {
1718         struct mlx5_core_dev *mdev = priv->mdev;
1719         u16 hw_mtu = 0;
1720         int err;
1721
1722         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
1723         if (err || !hw_mtu) /* fallback to port oper mtu */
1724                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
1725
1726         *mtu = MLX5E_HW2SW_MTU(hw_mtu);
1727 }
1728
1729 static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
1730 {
1731         struct mlx5e_priv *priv = netdev_priv(netdev);
1732         u16 mtu;
1733         int err;
1734
1735         err = mlx5e_set_mtu(priv, netdev->mtu);
1736         if (err)
1737                 return err;
1738
1739         mlx5e_query_mtu(priv, &mtu);
1740         if (mtu != netdev->mtu)
1741                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
1742                             __func__, mtu, netdev->mtu);
1743
1744         netdev->mtu = mtu;
1745         return 0;
1746 }
1747
1748 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
1749 {
1750         struct mlx5e_priv *priv = netdev_priv(netdev);
1751         int nch = priv->params.num_channels;
1752         int ntc = priv->params.num_tc;
1753         int tc;
1754
1755         netdev_reset_tc(netdev);
1756
1757         if (ntc == 1)
1758                 return;
1759
1760         netdev_set_num_tc(netdev, ntc);
1761
1762         for (tc = 0; tc < ntc; tc++)
1763                 netdev_set_tc_queue(netdev, tc, nch, tc * nch);
1764 }
1765
1766 int mlx5e_open_locked(struct net_device *netdev)
1767 {
1768         struct mlx5e_priv *priv = netdev_priv(netdev);
1769         int num_txqs;
1770         int err;
1771
1772         set_bit(MLX5E_STATE_OPENED, &priv->state);
1773
1774         mlx5e_netdev_set_tcs(netdev);
1775
1776         num_txqs = priv->params.num_channels * priv->params.num_tc;
1777         netif_set_real_num_tx_queues(netdev, num_txqs);
1778         netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
1779
1780         err = mlx5e_set_dev_port_mtu(netdev);
1781         if (err)
1782                 goto err_clear_state_opened_flag;
1783
1784         err = mlx5e_open_channels(priv);
1785         if (err) {
1786                 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
1787                            __func__, err);
1788                 goto err_clear_state_opened_flag;
1789         }
1790
1791         err = mlx5e_refresh_tirs_self_loopback_enable(priv);
1792         if (err) {
1793                 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
1794                            __func__, err);
1795                 goto err_close_channels;
1796         }
1797
1798         mlx5e_redirect_rqts(priv);
1799         mlx5e_update_carrier(priv);
1800         mlx5e_timestamp_init(priv);
1801 #ifdef CONFIG_RFS_ACCEL
1802         priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
1803 #endif
1804
1805         queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
1806
1807         return 0;
1808
1809 err_close_channels:
1810         mlx5e_close_channels(priv);
1811 err_clear_state_opened_flag:
1812         clear_bit(MLX5E_STATE_OPENED, &priv->state);
1813         return err;
1814 }
1815
1816 static int mlx5e_open(struct net_device *netdev)
1817 {
1818         struct mlx5e_priv *priv = netdev_priv(netdev);
1819         int err;
1820
1821         mutex_lock(&priv->state_lock);
1822         err = mlx5e_open_locked(netdev);
1823         mutex_unlock(&priv->state_lock);
1824
1825         return err;
1826 }
1827
1828 int mlx5e_close_locked(struct net_device *netdev)
1829 {
1830         struct mlx5e_priv *priv = netdev_priv(netdev);
1831
1832         /* May already be CLOSED in case a previous configuration operation
1833          * (e.g RX/TX queue size change) that involves close&open failed.
1834          */
1835         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
1836                 return 0;
1837
1838         clear_bit(MLX5E_STATE_OPENED, &priv->state);
1839
1840         mlx5e_timestamp_cleanup(priv);
1841         netif_carrier_off(priv->netdev);
1842         mlx5e_redirect_rqts(priv);
1843         mlx5e_close_channels(priv);
1844
1845         return 0;
1846 }
1847
1848 static int mlx5e_close(struct net_device *netdev)
1849 {
1850         struct mlx5e_priv *priv = netdev_priv(netdev);
1851         int err;
1852
1853         mutex_lock(&priv->state_lock);
1854         err = mlx5e_close_locked(netdev);
1855         mutex_unlock(&priv->state_lock);
1856
1857         return err;
1858 }
1859
1860 static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
1861                                 struct mlx5e_rq *rq,
1862                                 struct mlx5e_rq_param *param)
1863 {
1864         struct mlx5_core_dev *mdev = priv->mdev;
1865         void *rqc = param->rqc;
1866         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1867         int err;
1868
1869         param->wq.db_numa_node = param->wq.buf_numa_node;
1870
1871         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1872                                 &rq->wq_ctrl);
1873         if (err)
1874                 return err;
1875
1876         rq->priv = priv;
1877
1878         return 0;
1879 }
1880
1881 static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
1882                                 struct mlx5e_cq *cq,
1883                                 struct mlx5e_cq_param *param)
1884 {
1885         struct mlx5_core_dev *mdev = priv->mdev;
1886         struct mlx5_core_cq *mcq = &cq->mcq;
1887         int eqn_not_used;
1888         unsigned int irqn;
1889         int err;
1890
1891         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1892                                &cq->wq_ctrl);
1893         if (err)
1894                 return err;
1895
1896         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1897
1898         mcq->cqe_sz     = 64;
1899         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1900         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1901         *mcq->set_ci_db = 0;
1902         *mcq->arm_db    = 0;
1903         mcq->vector     = param->eq_ix;
1904         mcq->comp       = mlx5e_completion_event;
1905         mcq->event      = mlx5e_cq_error_event;
1906         mcq->irqn       = irqn;
1907         mcq->uar        = &priv->cq_uar;
1908
1909         cq->priv = priv;
1910
1911         return 0;
1912 }
1913
1914 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
1915 {
1916         struct mlx5e_cq_param cq_param;
1917         struct mlx5e_rq_param rq_param;
1918         struct mlx5e_rq *rq = &priv->drop_rq;
1919         struct mlx5e_cq *cq = &priv->drop_rq.cq;
1920         int err;
1921
1922         memset(&cq_param, 0, sizeof(cq_param));
1923         memset(&rq_param, 0, sizeof(rq_param));
1924         mlx5e_build_drop_rq_param(&rq_param);
1925
1926         err = mlx5e_create_drop_cq(priv, cq, &cq_param);
1927         if (err)
1928                 return err;
1929
1930         err = mlx5e_enable_cq(cq, &cq_param);
1931         if (err)
1932                 goto err_destroy_cq;
1933
1934         err = mlx5e_create_drop_rq(priv, rq, &rq_param);
1935         if (err)
1936                 goto err_disable_cq;
1937
1938         err = mlx5e_enable_rq(rq, &rq_param);
1939         if (err)
1940                 goto err_destroy_rq;
1941
1942         return 0;
1943
1944 err_destroy_rq:
1945         mlx5e_destroy_rq(&priv->drop_rq);
1946
1947 err_disable_cq:
1948         mlx5e_disable_cq(&priv->drop_rq.cq);
1949
1950 err_destroy_cq:
1951         mlx5e_destroy_cq(&priv->drop_rq.cq);
1952
1953         return err;
1954 }
1955
1956 static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
1957 {
1958         mlx5e_disable_rq(&priv->drop_rq);
1959         mlx5e_destroy_rq(&priv->drop_rq);
1960         mlx5e_disable_cq(&priv->drop_rq.cq);
1961         mlx5e_destroy_cq(&priv->drop_rq.cq);
1962 }
1963
1964 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
1965 {
1966         struct mlx5_core_dev *mdev = priv->mdev;
1967         u32 in[MLX5_ST_SZ_DW(create_tis_in)];
1968         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1969
1970         memset(in, 0, sizeof(in));
1971
1972         MLX5_SET(tisc, tisc, prio, tc << 1);
1973         MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
1974
1975         return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
1976 }
1977
1978 static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
1979 {
1980         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
1981 }
1982
1983 static int mlx5e_create_tises(struct mlx5e_priv *priv)
1984 {
1985         int err;
1986         int tc;
1987
1988         for (tc = 0; tc < MLX5E_MAX_NUM_TC; tc++) {
1989                 err = mlx5e_create_tis(priv, tc);
1990                 if (err)
1991                         goto err_close_tises;
1992         }
1993
1994         return 0;
1995
1996 err_close_tises:
1997         for (tc--; tc >= 0; tc--)
1998                 mlx5e_destroy_tis(priv, tc);
1999
2000         return err;
2001 }
2002
2003 static void mlx5e_destroy_tises(struct mlx5e_priv *priv)
2004 {
2005         int tc;
2006
2007         for (tc = 0; tc < MLX5E_MAX_NUM_TC; tc++)
2008                 mlx5e_destroy_tis(priv, tc);
2009 }
2010
2011 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2012                                       enum mlx5e_traffic_types tt)
2013 {
2014         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2015
2016         MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2017
2018 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2019                                  MLX5_HASH_FIELD_SEL_DST_IP)
2020
2021 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2022                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2023                                  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2024                                  MLX5_HASH_FIELD_SEL_L4_DPORT)
2025
2026 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2027                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2028                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2029
2030         mlx5e_build_tir_ctx_lro(tirc, priv);
2031
2032         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2033         MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqtn);
2034         mlx5e_build_tir_ctx_hash(tirc, priv);
2035
2036         switch (tt) {
2037         case MLX5E_TT_IPV4_TCP:
2038                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2039                          MLX5_L3_PROT_TYPE_IPV4);
2040                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2041                          MLX5_L4_PROT_TYPE_TCP);
2042                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2043                          MLX5_HASH_IP_L4PORTS);
2044                 break;
2045
2046         case MLX5E_TT_IPV6_TCP:
2047                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2048                          MLX5_L3_PROT_TYPE_IPV6);
2049                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2050                          MLX5_L4_PROT_TYPE_TCP);
2051                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2052                          MLX5_HASH_IP_L4PORTS);
2053                 break;
2054
2055         case MLX5E_TT_IPV4_UDP:
2056                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2057                          MLX5_L3_PROT_TYPE_IPV4);
2058                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2059                          MLX5_L4_PROT_TYPE_UDP);
2060                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2061                          MLX5_HASH_IP_L4PORTS);
2062                 break;
2063
2064         case MLX5E_TT_IPV6_UDP:
2065                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2066                          MLX5_L3_PROT_TYPE_IPV6);
2067                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2068                          MLX5_L4_PROT_TYPE_UDP);
2069                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2070                          MLX5_HASH_IP_L4PORTS);
2071                 break;
2072
2073         case MLX5E_TT_IPV4_IPSEC_AH:
2074                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2075                          MLX5_L3_PROT_TYPE_IPV4);
2076                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2077                          MLX5_HASH_IP_IPSEC_SPI);
2078                 break;
2079
2080         case MLX5E_TT_IPV6_IPSEC_AH:
2081                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2082                          MLX5_L3_PROT_TYPE_IPV6);
2083                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2084                          MLX5_HASH_IP_IPSEC_SPI);
2085                 break;
2086
2087         case MLX5E_TT_IPV4_IPSEC_ESP:
2088                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2089                          MLX5_L3_PROT_TYPE_IPV4);
2090                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2091                          MLX5_HASH_IP_IPSEC_SPI);
2092                 break;
2093
2094         case MLX5E_TT_IPV6_IPSEC_ESP:
2095                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2096                          MLX5_L3_PROT_TYPE_IPV6);
2097                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2098                          MLX5_HASH_IP_IPSEC_SPI);
2099                 break;
2100
2101         case MLX5E_TT_IPV4:
2102                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2103                          MLX5_L3_PROT_TYPE_IPV4);
2104                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2105                          MLX5_HASH_IP);
2106                 break;
2107
2108         case MLX5E_TT_IPV6:
2109                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2110                          MLX5_L3_PROT_TYPE_IPV6);
2111                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2112                          MLX5_HASH_IP);
2113                 break;
2114         default:
2115                 WARN_ONCE(true,
2116                           "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2117         }
2118 }
2119
2120 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2121                                        u32 rqtn)
2122 {
2123         MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2124
2125         mlx5e_build_tir_ctx_lro(tirc, priv);
2126
2127         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2128         MLX5_SET(tirc, tirc, indirect_table, rqtn);
2129         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2130 }
2131
2132 static int mlx5e_create_tirs(struct mlx5e_priv *priv)
2133 {
2134         int nch = mlx5e_get_max_num_channels(priv->mdev);
2135         void *tirc;
2136         int inlen;
2137         u32 *tirn;
2138         int err;
2139         u32 *in;
2140         int ix;
2141         int tt;
2142
2143         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2144         in = mlx5_vzalloc(inlen);
2145         if (!in)
2146                 return -ENOMEM;
2147
2148         /* indirect tirs */
2149         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2150                 memset(in, 0, inlen);
2151                 tirn = &priv->indir_tirn[tt];
2152                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2153                 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
2154                 err = mlx5_core_create_tir(priv->mdev, in, inlen, tirn);
2155                 if (err)
2156                         goto err_destroy_tirs;
2157         }
2158
2159         /* direct tirs */
2160         for (ix = 0; ix < nch; ix++) {
2161                 memset(in, 0, inlen);
2162                 tirn = &priv->direct_tir[ix].tirn;
2163                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2164                 mlx5e_build_direct_tir_ctx(priv, tirc,
2165                                            priv->direct_tir[ix].rqtn);
2166                 err = mlx5_core_create_tir(priv->mdev, in, inlen, tirn);
2167                 if (err)
2168                         goto err_destroy_ch_tirs;
2169         }
2170
2171         kvfree(in);
2172
2173         return 0;
2174
2175 err_destroy_ch_tirs:
2176         for (ix--; ix >= 0; ix--)
2177                 mlx5_core_destroy_tir(priv->mdev, priv->direct_tir[ix].tirn);
2178
2179 err_destroy_tirs:
2180         for (tt--; tt >= 0; tt--)
2181                 mlx5_core_destroy_tir(priv->mdev, priv->indir_tirn[tt]);
2182
2183         kvfree(in);
2184
2185         return err;
2186 }
2187
2188 static void mlx5e_destroy_tirs(struct mlx5e_priv *priv)
2189 {
2190         int nch = mlx5e_get_max_num_channels(priv->mdev);
2191         int i;
2192
2193         for (i = 0; i < nch; i++)
2194                 mlx5_core_destroy_tir(priv->mdev, priv->direct_tir[i].tirn);
2195
2196         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
2197                 mlx5_core_destroy_tir(priv->mdev, priv->indir_tirn[i]);
2198 }
2199
2200 int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2201 {
2202         int err = 0;
2203         int i;
2204
2205         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2206                 return 0;
2207
2208         for (i = 0; i < priv->params.num_channels; i++) {
2209                 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2210                 if (err)
2211                         return err;
2212         }
2213
2214         return 0;
2215 }
2216
2217 static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2218 {
2219         struct mlx5e_priv *priv = netdev_priv(netdev);
2220         bool was_opened;
2221         int err = 0;
2222
2223         if (tc && tc != MLX5E_MAX_NUM_TC)
2224                 return -EINVAL;
2225
2226         mutex_lock(&priv->state_lock);
2227
2228         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2229         if (was_opened)
2230                 mlx5e_close_locked(priv->netdev);
2231
2232         priv->params.num_tc = tc ? tc : 1;
2233
2234         if (was_opened)
2235                 err = mlx5e_open_locked(priv->netdev);
2236
2237         mutex_unlock(&priv->state_lock);
2238
2239         return err;
2240 }
2241
2242 static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2243                               __be16 proto, struct tc_to_netdev *tc)
2244 {
2245         struct mlx5e_priv *priv = netdev_priv(dev);
2246
2247         if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2248                 goto mqprio;
2249
2250         switch (tc->type) {
2251         case TC_SETUP_CLSFLOWER:
2252                 switch (tc->cls_flower->command) {
2253                 case TC_CLSFLOWER_REPLACE:
2254                         return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2255                 case TC_CLSFLOWER_DESTROY:
2256                         return mlx5e_delete_flower(priv, tc->cls_flower);
2257                 case TC_CLSFLOWER_STATS:
2258                         return mlx5e_stats_flower(priv, tc->cls_flower);
2259                 }
2260         default:
2261                 return -EOPNOTSUPP;
2262         }
2263
2264 mqprio:
2265         if (tc->type != TC_SETUP_MQPRIO)
2266                 return -EINVAL;
2267
2268         return mlx5e_setup_tc(dev, tc->tc);
2269 }
2270
2271 static struct rtnl_link_stats64 *
2272 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2273 {
2274         struct mlx5e_priv *priv = netdev_priv(dev);
2275         struct mlx5e_sw_stats *sstats = &priv->stats.sw;
2276         struct mlx5e_vport_stats *vstats = &priv->stats.vport;
2277         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
2278
2279         stats->rx_packets = sstats->rx_packets;
2280         stats->rx_bytes   = sstats->rx_bytes;
2281         stats->tx_packets = sstats->tx_packets;
2282         stats->tx_bytes   = sstats->tx_bytes;
2283
2284         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
2285         stats->tx_dropped = sstats->tx_queue_dropped;
2286
2287         stats->rx_length_errors =
2288                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2289                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2290                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
2291         stats->rx_crc_errors =
2292                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2293         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2294         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
2295         stats->tx_carrier_errors =
2296                 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
2297         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2298                            stats->rx_frame_errors;
2299         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2300
2301         /* vport multicast also counts packets that are dropped due to steering
2302          * or rx out of buffer
2303          */
2304         stats->multicast =
2305                 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
2306
2307         return stats;
2308 }
2309
2310 static void mlx5e_set_rx_mode(struct net_device *dev)
2311 {
2312         struct mlx5e_priv *priv = netdev_priv(dev);
2313
2314         queue_work(priv->wq, &priv->set_rx_mode_work);
2315 }
2316
2317 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2318 {
2319         struct mlx5e_priv *priv = netdev_priv(netdev);
2320         struct sockaddr *saddr = addr;
2321
2322         if (!is_valid_ether_addr(saddr->sa_data))
2323                 return -EADDRNOTAVAIL;
2324
2325         netif_addr_lock_bh(netdev);
2326         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2327         netif_addr_unlock_bh(netdev);
2328
2329         queue_work(priv->wq, &priv->set_rx_mode_work);
2330
2331         return 0;
2332 }
2333
2334 #define MLX5E_SET_FEATURE(netdev, feature, enable)      \
2335         do {                                            \
2336                 if (enable)                             \
2337                         netdev->features |= feature;    \
2338                 else                                    \
2339                         netdev->features &= ~feature;   \
2340         } while (0)
2341
2342 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2343
2344 static int set_feature_lro(struct net_device *netdev, bool enable)
2345 {
2346         struct mlx5e_priv *priv = netdev_priv(netdev);
2347         bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2348         int err;
2349
2350         mutex_lock(&priv->state_lock);
2351
2352         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2353                 mlx5e_close_locked(priv->netdev);
2354
2355         priv->params.lro_en = enable;
2356         err = mlx5e_modify_tirs_lro(priv);
2357         if (err) {
2358                 netdev_err(netdev, "lro modify failed, %d\n", err);
2359                 priv->params.lro_en = !enable;
2360         }
2361
2362         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2363                 mlx5e_open_locked(priv->netdev);
2364
2365         mutex_unlock(&priv->state_lock);
2366
2367         return err;
2368 }
2369
2370 static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2371 {
2372         struct mlx5e_priv *priv = netdev_priv(netdev);
2373
2374         if (enable)
2375                 mlx5e_enable_vlan_filter(priv);
2376         else
2377                 mlx5e_disable_vlan_filter(priv);
2378
2379         return 0;
2380 }
2381
2382 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2383 {
2384         struct mlx5e_priv *priv = netdev_priv(netdev);
2385
2386         if (!enable && mlx5e_tc_num_filters(priv)) {
2387                 netdev_err(netdev,
2388                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2389                 return -EINVAL;
2390         }
2391
2392         return 0;
2393 }
2394
2395 static int set_feature_rx_all(struct net_device *netdev, bool enable)
2396 {
2397         struct mlx5e_priv *priv = netdev_priv(netdev);
2398         struct mlx5_core_dev *mdev = priv->mdev;
2399
2400         return mlx5_set_port_fcs(mdev, !enable);
2401 }
2402
2403 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2404 {
2405         struct mlx5e_priv *priv = netdev_priv(netdev);
2406         int err;
2407
2408         mutex_lock(&priv->state_lock);
2409
2410         priv->params.vlan_strip_disable = !enable;
2411         err = mlx5e_modify_rqs_vsd(priv, !enable);
2412         if (err)
2413                 priv->params.vlan_strip_disable = enable;
2414
2415         mutex_unlock(&priv->state_lock);
2416
2417         return err;
2418 }
2419
2420 #ifdef CONFIG_RFS_ACCEL
2421 static int set_feature_arfs(struct net_device *netdev, bool enable)
2422 {
2423         struct mlx5e_priv *priv = netdev_priv(netdev);
2424         int err;
2425
2426         if (enable)
2427                 err = mlx5e_arfs_enable(priv);
2428         else
2429                 err = mlx5e_arfs_disable(priv);
2430
2431         return err;
2432 }
2433 #endif
2434
2435 static int mlx5e_handle_feature(struct net_device *netdev,
2436                                 netdev_features_t wanted_features,
2437                                 netdev_features_t feature,
2438                                 mlx5e_feature_handler feature_handler)
2439 {
2440         netdev_features_t changes = wanted_features ^ netdev->features;
2441         bool enable = !!(wanted_features & feature);
2442         int err;
2443
2444         if (!(changes & feature))
2445                 return 0;
2446
2447         err = feature_handler(netdev, enable);
2448         if (err) {
2449                 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2450                            enable ? "Enable" : "Disable", feature, err);
2451                 return err;
2452         }
2453
2454         MLX5E_SET_FEATURE(netdev, feature, enable);
2455         return 0;
2456 }
2457
2458 static int mlx5e_set_features(struct net_device *netdev,
2459                               netdev_features_t features)
2460 {
2461         int err;
2462
2463         err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2464                                     set_feature_lro);
2465         err |= mlx5e_handle_feature(netdev, features,
2466                                     NETIF_F_HW_VLAN_CTAG_FILTER,
2467                                     set_feature_vlan_filter);
2468         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2469                                     set_feature_tc_num_filters);
2470         err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2471                                     set_feature_rx_all);
2472         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2473                                     set_feature_rx_vlan);
2474 #ifdef CONFIG_RFS_ACCEL
2475         err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2476                                     set_feature_arfs);
2477 #endif
2478
2479         return err ? -EINVAL : 0;
2480 }
2481
2482 #define MXL5_HW_MIN_MTU 64
2483 #define MXL5E_MIN_MTU (MXL5_HW_MIN_MTU + ETH_FCS_LEN)
2484
2485 static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2486 {
2487         struct mlx5e_priv *priv = netdev_priv(netdev);
2488         struct mlx5_core_dev *mdev = priv->mdev;
2489         bool was_opened;
2490         u16 max_mtu;
2491         u16 min_mtu;
2492         int err = 0;
2493
2494         mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2495
2496         max_mtu = MLX5E_HW2SW_MTU(max_mtu);
2497         min_mtu = MLX5E_HW2SW_MTU(MXL5E_MIN_MTU);
2498
2499         if (new_mtu > max_mtu || new_mtu < min_mtu) {
2500                 netdev_err(netdev,
2501                            "%s: Bad MTU (%d), valid range is: [%d..%d]\n",
2502                            __func__, new_mtu, min_mtu, max_mtu);
2503                 return -EINVAL;
2504         }
2505
2506         mutex_lock(&priv->state_lock);
2507
2508         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2509         if (was_opened)
2510                 mlx5e_close_locked(netdev);
2511
2512         netdev->mtu = new_mtu;
2513
2514         if (was_opened)
2515                 err = mlx5e_open_locked(netdev);
2516
2517         mutex_unlock(&priv->state_lock);
2518
2519         return err;
2520 }
2521
2522 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2523 {
2524         switch (cmd) {
2525         case SIOCSHWTSTAMP:
2526                 return mlx5e_hwstamp_set(dev, ifr);
2527         case SIOCGHWTSTAMP:
2528                 return mlx5e_hwstamp_get(dev, ifr);
2529         default:
2530                 return -EOPNOTSUPP;
2531         }
2532 }
2533
2534 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2535 {
2536         struct mlx5e_priv *priv = netdev_priv(dev);
2537         struct mlx5_core_dev *mdev = priv->mdev;
2538
2539         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2540 }
2541
2542 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2543 {
2544         struct mlx5e_priv *priv = netdev_priv(dev);
2545         struct mlx5_core_dev *mdev = priv->mdev;
2546
2547         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2548                                            vlan, qos);
2549 }
2550
2551 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2552 {
2553         struct mlx5e_priv *priv = netdev_priv(dev);
2554         struct mlx5_core_dev *mdev = priv->mdev;
2555
2556         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2557 }
2558
2559 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2560 {
2561         struct mlx5e_priv *priv = netdev_priv(dev);
2562         struct mlx5_core_dev *mdev = priv->mdev;
2563
2564         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2565 }
2566 static int mlx5_vport_link2ifla(u8 esw_link)
2567 {
2568         switch (esw_link) {
2569         case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
2570                 return IFLA_VF_LINK_STATE_DISABLE;
2571         case MLX5_ESW_VPORT_ADMIN_STATE_UP:
2572                 return IFLA_VF_LINK_STATE_ENABLE;
2573         }
2574         return IFLA_VF_LINK_STATE_AUTO;
2575 }
2576
2577 static int mlx5_ifla_link2vport(u8 ifla_link)
2578 {
2579         switch (ifla_link) {
2580         case IFLA_VF_LINK_STATE_DISABLE:
2581                 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
2582         case IFLA_VF_LINK_STATE_ENABLE:
2583                 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
2584         }
2585         return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
2586 }
2587
2588 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
2589                                    int link_state)
2590 {
2591         struct mlx5e_priv *priv = netdev_priv(dev);
2592         struct mlx5_core_dev *mdev = priv->mdev;
2593
2594         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
2595                                             mlx5_ifla_link2vport(link_state));
2596 }
2597
2598 static int mlx5e_get_vf_config(struct net_device *dev,
2599                                int vf, struct ifla_vf_info *ivi)
2600 {
2601         struct mlx5e_priv *priv = netdev_priv(dev);
2602         struct mlx5_core_dev *mdev = priv->mdev;
2603         int err;
2604
2605         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
2606         if (err)
2607                 return err;
2608         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
2609         return 0;
2610 }
2611
2612 static int mlx5e_get_vf_stats(struct net_device *dev,
2613                               int vf, struct ifla_vf_stats *vf_stats)
2614 {
2615         struct mlx5e_priv *priv = netdev_priv(dev);
2616         struct mlx5_core_dev *mdev = priv->mdev;
2617
2618         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
2619                                             vf_stats);
2620 }
2621
2622 static void mlx5e_add_vxlan_port(struct net_device *netdev,
2623                                  struct udp_tunnel_info *ti)
2624 {
2625         struct mlx5e_priv *priv = netdev_priv(netdev);
2626
2627         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2628                 return;
2629
2630         if (!mlx5e_vxlan_allowed(priv->mdev))
2631                 return;
2632
2633         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
2634 }
2635
2636 static void mlx5e_del_vxlan_port(struct net_device *netdev,
2637                                  struct udp_tunnel_info *ti)
2638 {
2639         struct mlx5e_priv *priv = netdev_priv(netdev);
2640
2641         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2642                 return;
2643
2644         if (!mlx5e_vxlan_allowed(priv->mdev))
2645                 return;
2646
2647         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
2648 }
2649
2650 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
2651                                                     struct sk_buff *skb,
2652                                                     netdev_features_t features)
2653 {
2654         struct udphdr *udph;
2655         u16 proto;
2656         u16 port = 0;
2657
2658         switch (vlan_get_protocol(skb)) {
2659         case htons(ETH_P_IP):
2660                 proto = ip_hdr(skb)->protocol;
2661                 break;
2662         case htons(ETH_P_IPV6):
2663                 proto = ipv6_hdr(skb)->nexthdr;
2664                 break;
2665         default:
2666                 goto out;
2667         }
2668
2669         if (proto == IPPROTO_UDP) {
2670                 udph = udp_hdr(skb);
2671                 port = be16_to_cpu(udph->dest);
2672         }
2673
2674         /* Verify if UDP port is being offloaded by HW */
2675         if (port && mlx5e_vxlan_lookup_port(priv, port))
2676                 return features;
2677
2678 out:
2679         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
2680         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2681 }
2682
2683 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
2684                                               struct net_device *netdev,
2685                                               netdev_features_t features)
2686 {
2687         struct mlx5e_priv *priv = netdev_priv(netdev);
2688
2689         features = vlan_features_check(skb, features);
2690         features = vxlan_features_check(skb, features);
2691
2692         /* Validate if the tunneled packet is being offloaded by HW */
2693         if (skb->encapsulation &&
2694             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
2695                 return mlx5e_vxlan_features_check(priv, skb, features);
2696
2697         return features;
2698 }
2699
2700 static const struct net_device_ops mlx5e_netdev_ops_basic = {
2701         .ndo_open                = mlx5e_open,
2702         .ndo_stop                = mlx5e_close,
2703         .ndo_start_xmit          = mlx5e_xmit,
2704         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
2705         .ndo_select_queue        = mlx5e_select_queue,
2706         .ndo_get_stats64         = mlx5e_get_stats,
2707         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
2708         .ndo_set_mac_address     = mlx5e_set_mac,
2709         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
2710         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
2711         .ndo_set_features        = mlx5e_set_features,
2712         .ndo_change_mtu          = mlx5e_change_mtu,
2713         .ndo_do_ioctl            = mlx5e_ioctl,
2714         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
2715 #ifdef CONFIG_RFS_ACCEL
2716         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
2717 #endif
2718 };
2719
2720 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
2721         .ndo_open                = mlx5e_open,
2722         .ndo_stop                = mlx5e_close,
2723         .ndo_start_xmit          = mlx5e_xmit,
2724         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
2725         .ndo_select_queue        = mlx5e_select_queue,
2726         .ndo_get_stats64         = mlx5e_get_stats,
2727         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
2728         .ndo_set_mac_address     = mlx5e_set_mac,
2729         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
2730         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
2731         .ndo_set_features        = mlx5e_set_features,
2732         .ndo_change_mtu          = mlx5e_change_mtu,
2733         .ndo_do_ioctl            = mlx5e_ioctl,
2734         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
2735         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
2736         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
2737         .ndo_features_check      = mlx5e_features_check,
2738 #ifdef CONFIG_RFS_ACCEL
2739         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
2740 #endif
2741         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
2742         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
2743         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
2744         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
2745         .ndo_get_vf_config       = mlx5e_get_vf_config,
2746         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
2747         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
2748 };
2749
2750 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
2751 {
2752         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
2753                 return -ENOTSUPP;
2754         if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
2755             !MLX5_CAP_GEN(mdev, nic_flow_table) ||
2756             !MLX5_CAP_ETH(mdev, csum_cap) ||
2757             !MLX5_CAP_ETH(mdev, max_lso_cap) ||
2758             !MLX5_CAP_ETH(mdev, vlan_cap) ||
2759             !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
2760             MLX5_CAP_FLOWTABLE(mdev,
2761                                flow_table_properties_nic_receive.max_ft_level)
2762                                < 3) {
2763                 mlx5_core_warn(mdev,
2764                                "Not creating net device, some required device capabilities are missing\n");
2765                 return -ENOTSUPP;
2766         }
2767         if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
2768                 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
2769         if (!MLX5_CAP_GEN(mdev, cq_moderation))
2770                 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
2771
2772         return 0;
2773 }
2774
2775 u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
2776 {
2777         int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
2778
2779         return bf_buf_size -
2780                sizeof(struct mlx5e_tx_wqe) +
2781                2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
2782 }
2783
2784 #ifdef CONFIG_MLX5_CORE_EN_DCB
2785 static void mlx5e_ets_init(struct mlx5e_priv *priv)
2786 {
2787         int i;
2788
2789         priv->params.ets.ets_cap = mlx5_max_tc(priv->mdev) + 1;
2790         for (i = 0; i < priv->params.ets.ets_cap; i++) {
2791                 priv->params.ets.tc_tx_bw[i] = MLX5E_MAX_BW_ALLOC;
2792                 priv->params.ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR;
2793                 priv->params.ets.prio_tc[i] = i;
2794         }
2795
2796         /* tclass[prio=0]=1, tclass[prio=1]=0, tclass[prio=i]=i (for i>1) */
2797         priv->params.ets.prio_tc[0] = 1;
2798         priv->params.ets.prio_tc[1] = 0;
2799 }
2800 #endif
2801
2802 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
2803                                    u32 *indirection_rqt, int len,
2804                                    int num_channels)
2805 {
2806         int node = mdev->priv.numa_node;
2807         int node_num_of_cores;
2808         int i;
2809
2810         if (node == -1)
2811                 node = first_online_node;
2812
2813         node_num_of_cores = cpumask_weight(cpumask_of_node(node));
2814
2815         if (node_num_of_cores)
2816                 num_channels = min_t(int, num_channels, node_num_of_cores);
2817
2818         for (i = 0; i < len; i++)
2819                 indirection_rqt[i] = i % num_channels;
2820 }
2821
2822 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
2823 {
2824         return MLX5_CAP_GEN(mdev, striding_rq) &&
2825                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
2826                 MLX5_CAP_ETH(mdev, reg_umr_sq);
2827 }
2828
2829 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
2830 {
2831         enum pcie_link_width width;
2832         enum pci_bus_speed speed;
2833         int err = 0;
2834
2835         err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
2836         if (err)
2837                 return err;
2838
2839         if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
2840                 return -EINVAL;
2841
2842         switch (speed) {
2843         case PCIE_SPEED_2_5GT:
2844                 *pci_bw = 2500 * width;
2845                 break;
2846         case PCIE_SPEED_5_0GT:
2847                 *pci_bw = 5000 * width;
2848                 break;
2849         case PCIE_SPEED_8_0GT:
2850                 *pci_bw = 8000 * width;
2851                 break;
2852         default:
2853                 return -EINVAL;
2854         }
2855
2856         return 0;
2857 }
2858
2859 static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
2860 {
2861         return (link_speed && pci_bw &&
2862                 (pci_bw < 40000) && (pci_bw < link_speed));
2863 }
2864
2865 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
2866 {
2867         params->rx_cq_period_mode = cq_period_mode;
2868
2869         params->rx_cq_moderation.pkts =
2870                 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
2871         params->rx_cq_moderation.usec =
2872                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
2873
2874         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
2875                 params->rx_cq_moderation.usec =
2876                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
2877 }
2878
2879 static void mlx5e_build_netdev_priv(struct mlx5_core_dev *mdev,
2880                                     struct net_device *netdev,
2881                                     int num_channels)
2882 {
2883         struct mlx5e_priv *priv = netdev_priv(netdev);
2884         u32 link_speed = 0;
2885         u32 pci_bw = 0;
2886
2887         priv->params.log_sq_size           =
2888                 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
2889         priv->params.rq_wq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) ?
2890                 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
2891                 MLX5_WQ_TYPE_LINKED_LIST;
2892
2893         /* set CQE compression */
2894         priv->params.rx_cqe_compress_admin = false;
2895         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
2896             MLX5_CAP_GEN(mdev, vport_group_manager)) {
2897                 mlx5e_get_max_linkspeed(mdev, &link_speed);
2898                 mlx5e_get_pci_bw(mdev, &pci_bw);
2899                 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
2900                               link_speed, pci_bw);
2901                 priv->params.rx_cqe_compress_admin =
2902                         cqe_compress_heuristic(link_speed, pci_bw);
2903         }
2904
2905         priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
2906
2907         switch (priv->params.rq_wq_type) {
2908         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2909                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
2910                 priv->params.mpwqe_log_stride_sz =
2911                         priv->params.rx_cqe_compress ?
2912                         MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
2913                         MLX5_MPWRQ_LOG_STRIDE_SIZE;
2914                 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
2915                         priv->params.mpwqe_log_stride_sz;
2916                 priv->params.lro_en = true;
2917                 break;
2918         default: /* MLX5_WQ_TYPE_LINKED_LIST */
2919                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
2920         }
2921
2922         mlx5_core_info(mdev,
2923                        "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
2924                        priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
2925                        BIT(priv->params.log_rq_size),
2926                        BIT(priv->params.mpwqe_log_stride_sz),
2927                        priv->params.rx_cqe_compress_admin);
2928
2929         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
2930                                             BIT(priv->params.log_rq_size));
2931
2932         mlx5e_set_rx_cq_mode_params(&priv->params,
2933                                     MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2934
2935         priv->params.tx_cq_moderation.usec =
2936                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
2937         priv->params.tx_cq_moderation.pkts =
2938                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
2939         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
2940         priv->params.num_tc                = 1;
2941         priv->params.rss_hfunc             = ETH_RSS_HASH_XOR;
2942
2943         netdev_rss_key_fill(priv->params.toeplitz_hash_key,
2944                             sizeof(priv->params.toeplitz_hash_key));
2945
2946         mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
2947                                       MLX5E_INDIR_RQT_SIZE, num_channels);
2948
2949         priv->params.lro_wqe_sz            =
2950                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
2951
2952         /* Initialize pflags */
2953         MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
2954                             priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2955
2956         priv->mdev                         = mdev;
2957         priv->netdev                       = netdev;
2958         priv->params.num_channels          = num_channels;
2959
2960 #ifdef CONFIG_MLX5_CORE_EN_DCB
2961         mlx5e_ets_init(priv);
2962 #endif
2963
2964         mutex_init(&priv->state_lock);
2965
2966         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
2967         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
2968         INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
2969 }
2970
2971 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
2972 {
2973         struct mlx5e_priv *priv = netdev_priv(netdev);
2974
2975         mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
2976         if (is_zero_ether_addr(netdev->dev_addr) &&
2977             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
2978                 eth_hw_addr_random(netdev);
2979                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
2980         }
2981 }
2982
2983 static void mlx5e_build_netdev(struct net_device *netdev)
2984 {
2985         struct mlx5e_priv *priv = netdev_priv(netdev);
2986         struct mlx5_core_dev *mdev = priv->mdev;
2987         bool fcs_supported;
2988         bool fcs_enabled;
2989
2990         SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
2991
2992         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
2993                 netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
2994 #ifdef CONFIG_MLX5_CORE_EN_DCB
2995                 netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
2996 #endif
2997         } else {
2998                 netdev->netdev_ops = &mlx5e_netdev_ops_basic;
2999         }
3000
3001         netdev->watchdog_timeo    = 15 * HZ;
3002
3003         netdev->ethtool_ops       = &mlx5e_ethtool_ops;
3004
3005         netdev->vlan_features    |= NETIF_F_SG;
3006         netdev->vlan_features    |= NETIF_F_IP_CSUM;
3007         netdev->vlan_features    |= NETIF_F_IPV6_CSUM;
3008         netdev->vlan_features    |= NETIF_F_GRO;
3009         netdev->vlan_features    |= NETIF_F_TSO;
3010         netdev->vlan_features    |= NETIF_F_TSO6;
3011         netdev->vlan_features    |= NETIF_F_RXCSUM;
3012         netdev->vlan_features    |= NETIF_F_RXHASH;
3013
3014         if (!!MLX5_CAP_ETH(mdev, lro_cap))
3015                 netdev->vlan_features    |= NETIF_F_LRO;
3016
3017         netdev->hw_features       = netdev->vlan_features;
3018         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
3019         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
3020         netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
3021
3022         if (mlx5e_vxlan_allowed(mdev)) {
3023                 netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
3024                                            NETIF_F_GSO_UDP_TUNNEL_CSUM |
3025                                            NETIF_F_GSO_PARTIAL;
3026                 netdev->hw_enc_features |= NETIF_F_IP_CSUM;
3027                 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
3028                 netdev->hw_enc_features |= NETIF_F_TSO;
3029                 netdev->hw_enc_features |= NETIF_F_TSO6;
3030                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
3031                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3032                                            NETIF_F_GSO_PARTIAL;
3033                 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
3034         }
3035
3036         mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
3037
3038         if (fcs_supported)
3039                 netdev->hw_features |= NETIF_F_RXALL;
3040
3041         netdev->features          = netdev->hw_features;
3042         if (!priv->params.lro_en)
3043                 netdev->features  &= ~NETIF_F_LRO;
3044
3045         if (fcs_enabled)
3046                 netdev->features  &= ~NETIF_F_RXALL;
3047
3048 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
3049         if (FT_CAP(flow_modify_en) &&
3050             FT_CAP(modify_root) &&
3051             FT_CAP(identified_miss_table_mode) &&
3052             FT_CAP(flow_table_modify)) {
3053                 netdev->hw_features      |= NETIF_F_HW_TC;
3054 #ifdef CONFIG_RFS_ACCEL
3055                 netdev->hw_features      |= NETIF_F_NTUPLE;
3056 #endif
3057         }
3058
3059         netdev->features         |= NETIF_F_HIGHDMA;
3060
3061         netdev->priv_flags       |= IFF_UNICAST_FLT;
3062
3063         mlx5e_set_netdev_dev_addr(netdev);
3064 }
3065
3066 static int mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3067                              struct mlx5_core_mkey *mkey)
3068 {
3069         struct mlx5_core_dev *mdev = priv->mdev;
3070         struct mlx5_create_mkey_mbox_in *in;
3071         int err;
3072
3073         in = mlx5_vzalloc(sizeof(*in));
3074         if (!in)
3075                 return -ENOMEM;
3076
3077         in->seg.flags = MLX5_PERM_LOCAL_WRITE |
3078                         MLX5_PERM_LOCAL_READ  |
3079                         MLX5_ACCESS_MODE_PA;
3080         in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
3081         in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
3082
3083         err = mlx5_core_create_mkey(mdev, mkey, in, sizeof(*in), NULL, NULL,
3084                                     NULL);
3085
3086         kvfree(in);
3087
3088         return err;
3089 }
3090
3091 static void mlx5e_create_q_counter(struct mlx5e_priv *priv)
3092 {
3093         struct mlx5_core_dev *mdev = priv->mdev;
3094         int err;
3095
3096         err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
3097         if (err) {
3098                 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
3099                 priv->q_counter = 0;
3100         }
3101 }
3102
3103 static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
3104 {
3105         if (!priv->q_counter)
3106                 return;
3107
3108         mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
3109 }
3110
3111 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
3112 {
3113         struct mlx5_core_dev *mdev = priv->mdev;
3114         struct mlx5_create_mkey_mbox_in *in;
3115         struct mlx5_mkey_seg *mkc;
3116         int inlen = sizeof(*in);
3117         u64 npages =
3118                 mlx5e_get_max_num_channels(mdev) * MLX5_CHANNEL_MAX_NUM_MTTS;
3119         int err;
3120
3121         in = mlx5_vzalloc(inlen);
3122         if (!in)
3123                 return -ENOMEM;
3124
3125         mkc = &in->seg;
3126         mkc->status = MLX5_MKEY_STATUS_FREE;
3127         mkc->flags = MLX5_PERM_UMR_EN |
3128                      MLX5_PERM_LOCAL_READ |
3129                      MLX5_PERM_LOCAL_WRITE |
3130                      MLX5_ACCESS_MODE_MTT;
3131
3132         mkc->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
3133         mkc->flags_pd = cpu_to_be32(priv->pdn);
3134         mkc->len = cpu_to_be64(npages << PAGE_SHIFT);
3135         mkc->xlt_oct_size = cpu_to_be32(mlx5e_get_mtt_octw(npages));
3136         mkc->log2_page_size = PAGE_SHIFT;
3137
3138         err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen, NULL,
3139                                     NULL, NULL);
3140
3141         kvfree(in);
3142
3143         return err;
3144 }
3145
3146 static void *mlx5e_create_netdev(struct mlx5_core_dev *mdev)
3147 {
3148         struct net_device *netdev;
3149         struct mlx5e_priv *priv;
3150         int nch = mlx5e_get_max_num_channels(mdev);
3151         int err;
3152
3153         if (mlx5e_check_required_hca_cap(mdev))
3154                 return NULL;
3155
3156         netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
3157                                     nch * MLX5E_MAX_NUM_TC,
3158                                     nch);
3159         if (!netdev) {
3160                 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
3161                 return NULL;
3162         }
3163
3164         mlx5e_build_netdev_priv(mdev, netdev, nch);
3165         mlx5e_build_netdev(netdev);
3166
3167         netif_carrier_off(netdev);
3168
3169         priv = netdev_priv(netdev);
3170
3171         priv->wq = create_singlethread_workqueue("mlx5e");
3172         if (!priv->wq)
3173                 goto err_free_netdev;
3174
3175         err = mlx5_alloc_map_uar(mdev, &priv->cq_uar, false);
3176         if (err) {
3177                 mlx5_core_err(mdev, "alloc_map uar failed, %d\n", err);
3178                 goto err_destroy_wq;
3179         }
3180
3181         err = mlx5_core_alloc_pd(mdev, &priv->pdn);
3182         if (err) {
3183                 mlx5_core_err(mdev, "alloc pd failed, %d\n", err);
3184                 goto err_unmap_free_uar;
3185         }
3186
3187         err = mlx5_core_alloc_transport_domain(mdev, &priv->tdn);
3188         if (err) {
3189                 mlx5_core_err(mdev, "alloc td failed, %d\n", err);
3190                 goto err_dealloc_pd;
3191         }
3192
3193         err = mlx5e_create_mkey(priv, priv->pdn, &priv->mkey);
3194         if (err) {
3195                 mlx5_core_err(mdev, "create mkey failed, %d\n", err);
3196                 goto err_dealloc_transport_domain;
3197         }
3198
3199         err = mlx5e_create_umr_mkey(priv);
3200         if (err) {
3201                 mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
3202                 goto err_destroy_mkey;
3203         }
3204
3205         err = mlx5e_create_tises(priv);
3206         if (err) {
3207                 mlx5_core_warn(mdev, "create tises failed, %d\n", err);
3208                 goto err_destroy_umr_mkey;
3209         }
3210
3211         err = mlx5e_open_drop_rq(priv);
3212         if (err) {
3213                 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
3214                 goto err_destroy_tises;
3215         }
3216
3217         err = mlx5e_create_rqts(priv);
3218         if (err) {
3219                 mlx5_core_warn(mdev, "create rqts failed, %d\n", err);
3220                 goto err_close_drop_rq;
3221         }
3222
3223         err = mlx5e_create_tirs(priv);
3224         if (err) {
3225                 mlx5_core_warn(mdev, "create tirs failed, %d\n", err);
3226                 goto err_destroy_rqts;
3227         }
3228
3229         err = mlx5e_create_flow_steering(priv);
3230         if (err) {
3231                 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
3232                 goto err_destroy_tirs;
3233         }
3234
3235         mlx5e_create_q_counter(priv);
3236
3237         mlx5e_init_l2_addr(priv);
3238
3239         mlx5e_vxlan_init(priv);
3240
3241         err = mlx5e_tc_init(priv);
3242         if (err)
3243                 goto err_dealloc_q_counters;
3244
3245 #ifdef CONFIG_MLX5_CORE_EN_DCB
3246         mlx5e_dcbnl_ieee_setets_core(priv, &priv->params.ets);
3247 #endif
3248
3249         err = register_netdev(netdev);
3250         if (err) {
3251                 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
3252                 goto err_tc_cleanup;
3253         }
3254
3255         if (mlx5e_vxlan_allowed(mdev)) {
3256                 rtnl_lock();
3257                 udp_tunnel_get_rx_info(netdev);
3258                 rtnl_unlock();
3259         }
3260
3261         mlx5e_enable_async_events(priv);
3262         queue_work(priv->wq, &priv->set_rx_mode_work);
3263
3264         return priv;
3265
3266 err_tc_cleanup:
3267         mlx5e_tc_cleanup(priv);
3268
3269 err_dealloc_q_counters:
3270         mlx5e_destroy_q_counter(priv);
3271         mlx5e_destroy_flow_steering(priv);
3272
3273 err_destroy_tirs:
3274         mlx5e_destroy_tirs(priv);
3275
3276 err_destroy_rqts:
3277         mlx5e_destroy_rqts(priv);
3278
3279 err_close_drop_rq:
3280         mlx5e_close_drop_rq(priv);
3281
3282 err_destroy_tises:
3283         mlx5e_destroy_tises(priv);
3284
3285 err_destroy_umr_mkey:
3286         mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
3287
3288 err_destroy_mkey:
3289         mlx5_core_destroy_mkey(mdev, &priv->mkey);
3290
3291 err_dealloc_transport_domain:
3292         mlx5_core_dealloc_transport_domain(mdev, priv->tdn);
3293
3294 err_dealloc_pd:
3295         mlx5_core_dealloc_pd(mdev, priv->pdn);
3296
3297 err_unmap_free_uar:
3298         mlx5_unmap_free_uar(mdev, &priv->cq_uar);
3299
3300 err_destroy_wq:
3301         destroy_workqueue(priv->wq);
3302
3303 err_free_netdev:
3304         free_netdev(netdev);
3305
3306         return NULL;
3307 }
3308
3309 static void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, void *vpriv)
3310 {
3311         struct mlx5e_priv *priv = vpriv;
3312         struct net_device *netdev = priv->netdev;
3313
3314         set_bit(MLX5E_STATE_DESTROYING, &priv->state);
3315
3316         queue_work(priv->wq, &priv->set_rx_mode_work);
3317         mlx5e_disable_async_events(priv);
3318         flush_workqueue(priv->wq);
3319         if (test_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &mdev->intf_state)) {
3320                 netif_device_detach(netdev);
3321                 mlx5e_close(netdev);
3322         } else {
3323                 unregister_netdev(netdev);
3324         }
3325
3326         mlx5e_tc_cleanup(priv);
3327         mlx5e_vxlan_cleanup(priv);
3328         mlx5e_destroy_q_counter(priv);
3329         mlx5e_destroy_flow_steering(priv);
3330         mlx5e_destroy_tirs(priv);
3331         mlx5e_destroy_rqts(priv);
3332         mlx5e_close_drop_rq(priv);
3333         mlx5e_destroy_tises(priv);
3334         mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
3335         mlx5_core_destroy_mkey(priv->mdev, &priv->mkey);
3336         mlx5_core_dealloc_transport_domain(priv->mdev, priv->tdn);
3337         mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
3338         mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
3339         cancel_delayed_work_sync(&priv->update_stats_work);
3340         destroy_workqueue(priv->wq);
3341
3342         if (!test_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &mdev->intf_state))
3343                 free_netdev(netdev);
3344 }
3345
3346 static void *mlx5e_get_netdev(void *vpriv)
3347 {
3348         struct mlx5e_priv *priv = vpriv;
3349
3350         return priv->netdev;
3351 }
3352
3353 static struct mlx5_interface mlx5e_interface = {
3354         .add       = mlx5e_create_netdev,
3355         .remove    = mlx5e_destroy_netdev,
3356         .event     = mlx5e_async_event,
3357         .protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
3358         .get_dev   = mlx5e_get_netdev,
3359 };
3360
3361 void mlx5e_init(void)
3362 {
3363         mlx5_register_interface(&mlx5e_interface);
3364 }
3365
3366 void mlx5e_cleanup(void)
3367 {
3368         mlx5_unregister_interface(&mlx5e_interface);
3369 }