drivers/net/ethernet/mellanox/mlx4/en_rx.c

   1 /*
   2  * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/mlx4/cq.h>
  35 #include <linux/slab.h>
  36 #include <linux/mlx4/qp.h>
  37 #include <linux/skbuff.h>
  38 #include <linux/rculist.h>
  39 #include <linux/if_ether.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/vmalloc.h>
  42
  43 #include "mlx4_en.h"
  44
  45 static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
  46                                struct mlx4_en_rx_desc *rx_desc,
  47                                struct mlx4_en_rx_alloc *frags,
  48                                struct mlx4_en_rx_alloc *ring_alloc)
  49 {
  50         struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];
  51         struct mlx4_en_frag_info *frag_info;
  52         struct page *page;
  53         dma_addr_t dma;
  54         int i;
  55
  56         for (i = 0; i < priv->num_frags; i++) {
  57                 frag_info = &priv->frag_info[i];
  58                 if (ring_alloc[i].offset == frag_info->last_offset) {
  59                         page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
  60                                         MLX4_EN_ALLOC_ORDER);
  61                         if (!page)
  62                                 goto out;
  63                         dma = dma_map_page(priv->ddev, page, 0,
  64                                 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
  65                         if (dma_mapping_error(priv->ddev, dma)) {
  66                                 put_page(page);
  67                                 goto out;
  68                         }
  69                         page_alloc[i].page = page;
  70                         page_alloc[i].dma = dma;
  71                         page_alloc[i].offset = frag_info->frag_align;
  72                 } else {
  73                         page_alloc[i].page = ring_alloc[i].page;
  74                         get_page(ring_alloc[i].page);
  75                         page_alloc[i].dma = ring_alloc[i].dma;
  76                         page_alloc[i].offset = ring_alloc[i].offset +
  77                                                 frag_info->frag_stride;
  78                 }
  79         }
  80
  81         for (i = 0; i < priv->num_frags; i++) {
  82                 frags[i] = ring_alloc[i];
  83                 dma = ring_alloc[i].dma + ring_alloc[i].offset;
  84                 ring_alloc[i] = page_alloc[i];
  85                 rx_desc->data[i].addr = cpu_to_be64(dma);
  86         }
  87
  88         return 0;
  89
  90
  91 out:
  92         while (i--) {
  93                 frag_info = &priv->frag_info[i];
  94                 if (ring_alloc[i].offset == frag_info->last_offset)
  95                         dma_unmap_page(priv->ddev, page_alloc[i].dma,
  96                                 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
  97                 put_page(page_alloc[i].page);
  98         }
  99         return -ENOMEM;
 100 }
 101
 102 static void mlx4_en_free_frag(struct mlx4_en_priv *priv,
 103                               struct mlx4_en_rx_alloc *frags,
 104                               int i)
 105 {
 106         struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
 107
 108         if (frags[i].offset == frag_info->last_offset) {
 109                 dma_unmap_page(priv->ddev, frags[i].dma, MLX4_EN_ALLOC_SIZE,
 110                                          PCI_DMA_FROMDEVICE);
 111         }
 112         if (frags[i].page)
 113                 put_page(frags[i].page);
 114 }
 115
 116 static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
 117                                   struct mlx4_en_rx_ring *ring)
 118 {
 119         struct mlx4_en_rx_alloc *page_alloc;
 120         int i;
 121
 122         for (i = 0; i < priv->num_frags; i++) {
 123                 page_alloc = &ring->page_alloc[i];
 124                 page_alloc->page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
 125                                                MLX4_EN_ALLOC_ORDER);
 126                 if (!page_alloc->page)
 127                         goto out;
 128
 129                 page_alloc->dma = dma_map_page(priv->ddev, page_alloc->page, 0,
 130                                         MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
 131                 if (dma_mapping_error(priv->ddev, page_alloc->dma)) {
 132                         put_page(page_alloc->page);
 133                         page_alloc->page = NULL;
 134                         goto out;
 135                 }
 136                 page_alloc->offset = priv->frag_info[i].frag_align;
 137                 en_dbg(DRV, priv, "Initialized allocator:%d with page:%p\n",
 138                        i, page_alloc->page);
 139         }
 140         return 0;
 141
 142 out:
 143         while (i--) {
 144                 page_alloc = &ring->page_alloc[i];
 145                 dma_unmap_page(priv->ddev, page_alloc->dma,
 146                                 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
 147                 put_page(page_alloc->page);
 148                 page_alloc->page = NULL;
 149         }
 150         return -ENOMEM;
 151 }
 152
 153 static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
 154                                       struct mlx4_en_rx_ring *ring)
 155 {
 156         struct mlx4_en_rx_alloc *page_alloc;
 157         int i;
 158
 159         for (i = 0; i < priv->num_frags; i++) {
 160                 page_alloc = &ring->page_alloc[i];
 161                 en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
 162                        i, page_count(page_alloc->page));
 163
 164                 dma_unmap_page(priv->ddev, page_alloc->dma,
 165                                 MLX4_EN_ALLOC_SIZE, PCI_DMA_FROMDEVICE);
 166                 put_page(page_alloc->page);
 167                 page_alloc->page = NULL;
 168         }
 169 }
 170
 171 static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
 172                                  struct mlx4_en_rx_ring *ring, int index)
 173 {
 174         struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
 175         int possible_frags;
 176         int i;
 177
 178         /* Set size and memtype fields */
 179         for (i = 0; i < priv->num_frags; i++) {
 180                 rx_desc->data[i].byte_count =
 181                         cpu_to_be32(priv->frag_info[i].frag_size);
 182                 rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
 183         }
 184
 185         /* If the number of used fragments does not fill up the ring stride,
 186          * remaining (unused) fragments must be padded with null address/size
 187          * and a special memory key */
 188         possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
 189         for (i = priv->num_frags; i < possible_frags; i++) {
 190                 rx_desc->data[i].byte_count = 0;
 191                 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
 192                 rx_desc->data[i].addr = 0;
 193         }
 194 }
 195
 196 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
 197                                    struct mlx4_en_rx_ring *ring, int index)
 198 {
 199         struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
 200         struct mlx4_en_rx_alloc *frags = ring->rx_info +
 201                                         (index << priv->log_rx_info);
 202
 203         return mlx4_en_alloc_frags(priv, rx_desc, frags, ring->page_alloc);
 204 }
 205
 206 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
 207 {
 208         *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
 209 }
 210
 211 static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
 212                                  struct mlx4_en_rx_ring *ring,
 213                                  int index)
 214 {
 215         struct mlx4_en_rx_alloc *frags;
 216         int nr;
 217
 218         frags = ring->rx_info + (index << priv->log_rx_info);
 219         for (nr = 0; nr < priv->num_frags; nr++) {
 220                 en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
 221                 mlx4_en_free_frag(priv, frags, nr);
 222         }
 223 }
 224
 225 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
 226 {
 227         struct mlx4_en_rx_ring *ring;
 228         int ring_ind;
 229         int buf_ind;
 230         int new_size;
 231
 232         for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
 233                 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 234                         ring = &priv->rx_ring[ring_ind];
 235
 236                         if (mlx4_en_prepare_rx_desc(priv, ring,
 237                                                     ring->actual_size)) {
 238                                 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
 239                                         en_err(priv, "Failed to allocate "
 240                                                      "enough rx buffers\n");
 241                                         return -ENOMEM;
 242                                 } else {
 243                                         new_size = rounddown_pow_of_two(ring->actual_size);
 244                                         en_warn(priv, "Only %d buffers allocated "
 245                                                       "reducing ring size to %d",
 246                                                 ring->actual_size, new_size);
 247                                         goto reduce_rings;
 248                                 }
 249                         }
 250                         ring->actual_size++;
 251                         ring->prod++;
 252                 }
 253         }
 254         return 0;
 255
 256 reduce_rings:
 257         for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 258                 ring = &priv->rx_ring[ring_ind];
 259                 while (ring->actual_size > new_size) {
 260                         ring->actual_size--;
 261                         ring->prod--;
 262                         mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
 263                 }
 264         }
 265
 266         return 0;
 267 }
 268
 269 static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
 270                                 struct mlx4_en_rx_ring *ring)
 271 {
 272         int index;
 273
 274         en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
 275                ring->cons, ring->prod);
 276
 277         /* Unmap and free Rx buffers */
 278         BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);
 279         while (ring->cons != ring->prod) {
 280                 index = ring->cons & ring->size_mask;
 281                 en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
 282                 mlx4_en_free_rx_desc(priv, ring, index);
 283                 ++ring->cons;
 284         }
 285 }
 286
 287 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 288                            struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
 289 {
 290         struct mlx4_en_dev *mdev = priv->mdev;
 291         int err = -ENOMEM;
 292         int tmp;
 293
 294         ring->prod = 0;
 295         ring->cons = 0;
 296         ring->size = size;
 297         ring->size_mask = size - 1;
 298         ring->stride = stride;
 299         ring->log_stride = ffs(ring->stride) - 1;
 300         ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
 301
 302         tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
 303                                         sizeof(struct mlx4_en_rx_alloc));
 304         ring->rx_info = vmalloc(tmp);
 305         if (!ring->rx_info)
 306                 return -ENOMEM;
 307
 308         en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
 309                  ring->rx_info, tmp);
 310
 311         err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
 312                                  ring->buf_size, 2 * PAGE_SIZE);
 313         if (err)
 314                 goto err_ring;
 315
 316         err = mlx4_en_map_buffer(&ring->wqres.buf);
 317         if (err) {
 318                 en_err(priv, "Failed to map RX buffer\n");
 319                 goto err_hwq;
 320         }
 321         ring->buf = ring->wqres.buf.direct.buf;
 322
 323         ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;
 324
 325         return 0;
 326
 327 err_hwq:
 328         mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
 329 err_ring:
 330         vfree(ring->rx_info);
 331         ring->rx_info = NULL;
 332         return err;
 333 }
 334
 335 int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
 336 {
 337         struct mlx4_en_rx_ring *ring;
 338         int i;
 339         int ring_ind;
 340         int err;
 341         int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
 342                                         DS_SIZE * priv->num_frags);
 343
 344         for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 345                 ring = &priv->rx_ring[ring_ind];
 346
 347                 ring->prod = 0;
 348                 ring->cons = 0;
 349                 ring->actual_size = 0;
 350                 ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;
 351
 352                 ring->stride = stride;
 353                 if (ring->stride <= TXBB_SIZE)
 354                         ring->buf += TXBB_SIZE;
 355
 356                 ring->log_stride = ffs(ring->stride) - 1;
 357                 ring->buf_size = ring->size * ring->stride;
 358
 359                 memset(ring->buf, 0, ring->buf_size);
 360                 mlx4_en_update_rx_prod_db(ring);
 361
 362                 /* Initialize all descriptors */
 363                 for (i = 0; i < ring->size; i++)
 364                         mlx4_en_init_rx_desc(priv, ring, i);
 365
 366                 /* Initialize page allocators */
 367                 err = mlx4_en_init_allocator(priv, ring);
 368                 if (err) {
 369                         en_err(priv, "Failed initializing ring allocator\n");
 370                         if (ring->stride <= TXBB_SIZE)
 371                                 ring->buf -= TXBB_SIZE;
 372                         ring_ind--;
 373                         goto err_allocator;
 374                 }
 375         }
 376         err = mlx4_en_fill_rx_buffers(priv);
 377         if (err)
 378                 goto err_buffers;
 379
 380         for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 381                 ring = &priv->rx_ring[ring_ind];
 382
 383                 ring->size_mask = ring->actual_size - 1;
 384                 mlx4_en_update_rx_prod_db(ring);
 385         }
 386
 387         return 0;
 388
 389 err_buffers:
 390         for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
 391                 mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);
 392
 393         ring_ind = priv->rx_ring_num - 1;
 394 err_allocator:
 395         while (ring_ind >= 0) {
 396                 if (priv->rx_ring[ring_ind].stride <= TXBB_SIZE)
 397                         priv->rx_ring[ring_ind].buf -= TXBB_SIZE;
 398                 mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]);
 399                 ring_ind--;
 400         }
 401         return err;
 402 }
 403
 404 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
 405                              struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
 406 {
 407         struct mlx4_en_dev *mdev = priv->mdev;
 408
 409         mlx4_en_unmap_buffer(&ring->wqres.buf);
 410         mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
 411         vfree(ring->rx_info);
 412         ring->rx_info = NULL;
 413 #ifdef CONFIG_RFS_ACCEL
 414         mlx4_en_cleanup_filters(priv, ring);
 415 #endif
 416 }
 417
 418 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
 419                                 struct mlx4_en_rx_ring *ring)
 420 {
 421         mlx4_en_free_rx_buf(priv, ring);
 422         if (ring->stride <= TXBB_SIZE)
 423                 ring->buf -= TXBB_SIZE;
 424         mlx4_en_destroy_allocator(priv, ring);
 425 }
 426
 427
 428 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
 429                                     struct mlx4_en_rx_desc *rx_desc,
 430                                     struct mlx4_en_rx_alloc *frags,
 431                                     struct sk_buff *skb,
 432                                     int length)
 433 {
 434         struct skb_frag_struct *skb_frags_rx = skb_shinfo(skb)->frags;
 435         struct mlx4_en_frag_info *frag_info;
 436         int nr;
 437         dma_addr_t dma;
 438
 439         /* Collect used fragments while replacing them in the HW descriptors */
 440         for (nr = 0; nr < priv->num_frags; nr++) {
 441                 frag_info = &priv->frag_info[nr];
 442                 if (length <= frag_info->frag_prefix_size)
 443                         break;
 444                 if (!frags[nr].page)
 445                         goto fail;
 446
 447                 dma = be64_to_cpu(rx_desc->data[nr].addr);
 448                 dma_sync_single_for_cpu(priv->ddev, dma, frag_info->frag_size,
 449                                         DMA_FROM_DEVICE);
 450
 451                 /* Save page reference in skb */
 452                 get_page(frags[nr].page);
 453                 __skb_frag_set_page(&skb_frags_rx[nr], frags[nr].page);
 454                 skb_frag_size_set(&skb_frags_rx[nr], frag_info->frag_size);
 455                 skb_frags_rx[nr].page_offset = frags[nr].offset;
 456                 skb->truesize += frag_info->frag_stride;
 457         }
 458         /* Adjust size of last fragment to match actual length */
 459         if (nr > 0)
 460                 skb_frag_size_set(&skb_frags_rx[nr - 1],
 461                         length - priv->frag_info[nr - 1].frag_prefix_size);
 462         return nr;
 463
 464 fail:
 465         while (nr > 0) {
 466                 nr--;
 467                 __skb_frag_unref(&skb_frags_rx[nr]);
 468         }
 469         return 0;
 470 }
 471
 472
 473 static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
 474                                       struct mlx4_en_rx_desc *rx_desc,
 475                                       struct mlx4_en_rx_alloc *frags,
 476                                       unsigned int length)
 477 {
 478         struct sk_buff *skb;
 479         void *va;
 480         int used_frags;
 481         dma_addr_t dma;
 482
 483         skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
 484         if (!skb) {
 485                 en_dbg(RX_ERR, priv, "Failed allocating skb\n");
 486                 return NULL;
 487         }
 488         skb_reserve(skb, NET_IP_ALIGN);
 489         skb->len = length;
 490
 491         /* Get pointer to first fragment so we could copy the headers into the
 492          * (linear part of the) skb */
 493         va = page_address(frags[0].page) + frags[0].offset;
 494
 495         if (length <= SMALL_PACKET_SIZE) {
 496                 /* We are copying all relevant data to the skb - temporarily
 497                  * sync buffers for the copy */
 498                 dma = be64_to_cpu(rx_desc->data[0].addr);
 499                 dma_sync_single_for_cpu(priv->ddev, dma, length,
 500                                         DMA_FROM_DEVICE);
 501                 skb_copy_to_linear_data(skb, va, length);
 502                 skb->tail += length;
 503         } else {
 504                 /* Move relevant fragments to skb */
 505                 used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, frags,
 506                                                         skb, length);
 507                 if (unlikely(!used_frags)) {
 508                         kfree_skb(skb);
 509                         return NULL;
 510                 }
 511                 skb_shinfo(skb)->nr_frags = used_frags;
 512
 513                 /* Copy headers into the skb linear buffer */
 514                 memcpy(skb->data, va, HEADER_COPY_SIZE);
 515                 skb->tail += HEADER_COPY_SIZE;
 516
 517                 /* Skip headers in first fragment */
 518                 skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE;
 519
 520                 /* Adjust size of first fragment */
 521                 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], HEADER_COPY_SIZE);
 522                 skb->data_len = length - HEADER_COPY_SIZE;
 523         }
 524         return skb;
 525 }
 526
 527 static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
 528 {
 529         int i;
 530         int offset = ETH_HLEN;
 531
 532         for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
 533                 if (*(skb->data + offset) != (unsigned char) (i & 0xff))
 534                         goto out_loopback;
 535         }
 536         /* Loopback found */
 537         priv->loopback_ok = 1;
 538
 539 out_loopback:
 540         dev_kfree_skb_any(skb);
 541 }
 542
 543 static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
 544                                      struct mlx4_en_rx_ring *ring)
 545 {
 546         int index = ring->prod & ring->size_mask;
 547
 548         while ((u32) (ring->prod - ring->cons) < ring->actual_size) {
 549                 if (mlx4_en_prepare_rx_desc(priv, ring, index))
 550                         break;
 551                 ring->prod++;
 552                 index = ring->prod & ring->size_mask;
 553         }
 554 }
 555
 556 int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
 557 {
 558         struct mlx4_en_priv *priv = netdev_priv(dev);
 559         struct mlx4_en_dev *mdev = priv->mdev;
 560         struct mlx4_cqe *cqe;
 561         struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
 562         struct mlx4_en_rx_alloc *frags;
 563         struct mlx4_en_rx_desc *rx_desc;
 564         struct sk_buff *skb;
 565         int index;
 566         int nr;
 567         unsigned int length;
 568         int polled = 0;
 569         int ip_summed;
 570         int factor = priv->cqe_factor;
 571         u64 timestamp;
 572
 573         if (!priv->port_up)
 574                 return 0;
 575
 576         /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
 577          * descriptor offset can be deduced from the CQE index instead of
 578          * reading 'cqe->index' */
 579         index = cq->mcq.cons_index & ring->size_mask;
 580         cqe = &cq->buf[(index << factor) + factor];
 581
 582         /* Process all completed CQEs */
 583         while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
 584                     cq->mcq.cons_index & cq->size)) {
 585
 586                 frags = ring->rx_info + (index << priv->log_rx_info);
 587                 rx_desc = ring->buf + (index << ring->log_stride);
 588
 589                 /*
 590                  * make sure we read the CQE after we read the ownership bit
 591                  */
 592                 rmb();
 593
 594                 /* Drop packet on bad receive or bad checksum */
 595                 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
 596                                                 MLX4_CQE_OPCODE_ERROR)) {
 597                         en_err(priv, "CQE completed in error - vendor "
 598                                   "syndrom:%d syndrom:%d\n",
 599                                   ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
 600                                   ((struct mlx4_err_cqe *) cqe)->syndrome);
 601                         goto next;
 602                 }
 603                 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
 604                         en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
 605                         goto next;
 606                 }
 607
 608                 /* Check if we need to drop the packet if SRIOV is not enabled
 609                  * and not performing the selftest or flb disabled
 610                  */
 611                 if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
 612                         struct ethhdr *ethh;
 613                         dma_addr_t dma;
 614                         /* Get pointer to first fragment since we haven't
 615                          * skb yet and cast it to ethhdr struct
 616                          */
 617                         dma = be64_to_cpu(rx_desc->data[0].addr);
 618                         dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
 619                                                 DMA_FROM_DEVICE);
 620                         ethh = (struct ethhdr *)(page_address(frags[0].page) +
 621                                                  frags[0].offset);
 622
 623                         if (is_multicast_ether_addr(ethh->h_dest)) {
 624                                 struct mlx4_mac_entry *entry;
 625                                 struct hlist_head *bucket;
 626                                 unsigned int mac_hash;
 627
 628                                 /* Drop the packet, since HW loopback-ed it */
 629                                 mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
 630                                 bucket = &priv->mac_hash[mac_hash];
 631                                 rcu_read_lock();
 632                                 hlist_for_each_entry_rcu(entry, bucket, hlist) {
 633                                         if (ether_addr_equal_64bits(entry->mac,
 634                                                                     ethh->h_source)) {
 635                                                 rcu_read_unlock();
 636                                                 goto next;
 637                                         }
 638                                 }
 639                                 rcu_read_unlock();
 640                         }
 641                 }
 642
 643                 /*
 644                  * Packet is OK - process it.
 645                  */
 646                 length = be32_to_cpu(cqe->byte_cnt);
 647                 length -= ring->fcs_del;
 648                 ring->bytes += length;
 649                 ring->packets++;
 650
 651                 if (likely(dev->features & NETIF_F_RXCSUM)) {
 652                         if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
 653                             (cqe->checksum == cpu_to_be16(0xffff))) {
 654                                 ring->csum_ok++;
 655                                 /* This packet is eligible for GRO if it is:
 656                                  * - DIX Ethernet (type interpretation)
 657                                  * - TCP/IP (v4)
 658                                  * - without IP options
 659                                  * - not an IP fragment */
 660                                 if (dev->features & NETIF_F_GRO) {
 661                                         struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
 662                                         if (!gro_skb)
 663                                                 goto next;
 664
 665                                         nr = mlx4_en_complete_rx_desc(priv,
 666                                                 rx_desc, frags, gro_skb,
 667                                                 length);
 668                                         if (!nr)
 669                                                 goto next;
 670
 671                                         skb_shinfo(gro_skb)->nr_frags = nr;
 672                                         gro_skb->len = length;
 673                                         gro_skb->data_len = length;
 674                                         gro_skb->ip_summed = CHECKSUM_UNNECESSARY;
 675
 676                                         if ((cqe->vlan_my_qpn &
 677                                             cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)) &&
 678                                             (dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
 679                                                 u16 vid = be16_to_cpu(cqe->sl_vid);
 680
 681                                                 __vlan_hwaccel_put_tag(gro_skb, htons(ETH_P_8021Q), vid);
 682                                         }
 683
 684                                         if (dev->features & NETIF_F_RXHASH)
 685                                                 gro_skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid);
 686
 687                                         skb_record_rx_queue(gro_skb, cq->ring);
 688
 689                                         if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
 690                                                 timestamp = mlx4_en_get_cqe_ts(cqe);
 691                                                 mlx4_en_fill_hwtstamps(mdev,
 692                                                                        skb_hwtstamps(gro_skb),
 693                                                                        timestamp);
 694                                         }
 695
 696                                         napi_gro_frags(&cq->napi);
 697                                         goto next;
 698                                 }
 699
 700                                 /* GRO not possible, complete processing here */
 701                                 ip_summed = CHECKSUM_UNNECESSARY;
 702                         } else {
 703                                 ip_summed = CHECKSUM_NONE;
 704                                 ring->csum_none++;
 705                         }
 706                 } else {
 707                         ip_summed = CHECKSUM_NONE;
 708                         ring->csum_none++;
 709                 }
 710
 711                 skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
 712                 if (!skb) {
 713                         priv->stats.rx_dropped++;
 714                         goto next;
 715                 }
 716
 717                 if (unlikely(priv->validate_loopback)) {
 718                         validate_loopback(priv, skb);
 719                         goto next;
 720                 }
 721
 722                 skb->ip_summed = ip_summed;
 723                 skb->protocol = eth_type_trans(skb, dev);
 724                 skb_record_rx_queue(skb, cq->ring);
 725
 726                 if (dev->features & NETIF_F_RXHASH)
 727                         skb->rxhash = be32_to_cpu(cqe->immed_rss_invalid);
 728
 729                 if ((be32_to_cpu(cqe->vlan_my_qpn) &
 730                     MLX4_CQE_VLAN_PRESENT_MASK) &&
 731                     (dev->features & NETIF_F_HW_VLAN_CTAG_RX))
 732                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(cqe->sl_vid));
 733
 734                 if (ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL) {
 735                         timestamp = mlx4_en_get_cqe_ts(cqe);
 736                         mlx4_en_fill_hwtstamps(mdev, skb_hwtstamps(skb),
 737                                                timestamp);
 738                 }
 739
 740                 /* Push it up the stack */
 741                 netif_receive_skb(skb);
 742
 743 next:
 744                 for (nr = 0; nr < priv->num_frags; nr++)
 745                         mlx4_en_free_frag(priv, frags, nr);
 746
 747                 ++cq->mcq.cons_index;
 748                 index = (cq->mcq.cons_index) & ring->size_mask;
 749                 cqe = &cq->buf[(index << factor) + factor];
 750                 if (++polled == budget)
 751                         goto out;
 752         }
 753
 754 out:
 755         AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
 756         mlx4_cq_set_ci(&cq->mcq);
 757         wmb(); /* ensure HW sees CQ consumer before we post new buffers */
 758         ring->cons = cq->mcq.cons_index;
 759         mlx4_en_refill_rx_buffers(priv, ring);
 760         mlx4_en_update_rx_prod_db(ring);
 761         return polled;
 762 }
 763
 764
 765 void mlx4_en_rx_irq(struct mlx4_cq *mcq)
 766 {
 767         struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
 768         struct mlx4_en_priv *priv = netdev_priv(cq->dev);
 769
 770         if (priv->port_up)
 771                 napi_schedule(&cq->napi);
 772         else
 773                 mlx4_en_arm_cq(priv, cq);
 774 }
 775
 776 /* Rx CQ polling - called by NAPI */
 777 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
 778 {
 779         struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
 780         struct net_device *dev = cq->dev;
 781         struct mlx4_en_priv *priv = netdev_priv(dev);
 782         int done;
 783
 784         done = mlx4_en_process_rx_cq(dev, cq, budget);
 785
 786         /* If we used up all the quota - we're probably not done yet... */
 787         if (done == budget)
 788                 INC_PERF_COUNTER(priv->pstats.napi_quota);
 789         else {
 790                 /* Done for now */
 791                 napi_complete(napi);
 792                 mlx4_en_arm_cq(priv, cq);
 793         }
 794         return done;
 795 }
 796
 797
 798 /* Calculate the last offset position that accommodates a full fragment
 799  * (assuming fagment size = stride-align) */
 800 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv *priv, u16 stride, u16 align)
 801 {
 802         u16 res = MLX4_EN_ALLOC_SIZE % stride;
 803         u16 offset = MLX4_EN_ALLOC_SIZE - stride - res + align;
 804
 805         en_dbg(DRV, priv, "Calculated last offset for stride:%d align:%d "
 806                             "res:%d offset:%d\n", stride, align, res, offset);
 807         return offset;
 808 }
 809
 810
 811 static int frag_sizes[] = {
 812         FRAG_SZ0,
 813         FRAG_SZ1,
 814         FRAG_SZ2,
 815         FRAG_SZ3
 816 };
 817
 818 void mlx4_en_calc_rx_buf(struct net_device *dev)
 819 {
 820         struct mlx4_en_priv *priv = netdev_priv(dev);
 821         int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN + ETH_LLC_SNAP_SIZE;
 822         int buf_size = 0;
 823         int i = 0;
 824
 825         while (buf_size < eff_mtu) {
 826                 priv->frag_info[i].frag_size =
 827                         (eff_mtu > buf_size + frag_sizes[i]) ?
 828                                 frag_sizes[i] : eff_mtu - buf_size;
 829                 priv->frag_info[i].frag_prefix_size = buf_size;
 830                 if (!i) {
 831                         priv->frag_info[i].frag_align = NET_IP_ALIGN;
 832                         priv->frag_info[i].frag_stride =
 833                                 ALIGN(frag_sizes[i] + NET_IP_ALIGN, SMP_CACHE_BYTES);
 834                 } else {
 835                         priv->frag_info[i].frag_align = 0;
 836                         priv->frag_info[i].frag_stride =
 837                                 ALIGN(frag_sizes[i], SMP_CACHE_BYTES);
 838                 }
 839                 priv->frag_info[i].last_offset = mlx4_en_last_alloc_offset(
 840                                                 priv, priv->frag_info[i].frag_stride,
 841                                                 priv->frag_info[i].frag_align);
 842                 buf_size += priv->frag_info[i].frag_size;
 843                 i++;
 844         }
 845
 846         priv->num_frags = i;
 847         priv->rx_skb_size = eff_mtu;
 848         priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
 849
 850         en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d "
 851                   "num_frags:%d):\n", eff_mtu, priv->num_frags);
 852         for (i = 0; i < priv->num_frags; i++) {
 853                 en_dbg(DRV, priv, "  frag:%d - size:%d prefix:%d align:%d "
 854                                 "stride:%d last_offset:%d\n", i,
 855                                 priv->frag_info[i].frag_size,
 856                                 priv->frag_info[i].frag_prefix_size,
 857                                 priv->frag_info[i].frag_align,
 858                                 priv->frag_info[i].frag_stride,
 859                                 priv->frag_info[i].last_offset);
 860         }
 861 }
 862
 863 /* RSS related functions */
 864
 865 static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
 866                                  struct mlx4_en_rx_ring *ring,
 867                                  enum mlx4_qp_state *state,
 868                                  struct mlx4_qp *qp)
 869 {
 870         struct mlx4_en_dev *mdev = priv->mdev;
 871         struct mlx4_qp_context *context;
 872         int err = 0;
 873
 874         context = kmalloc(sizeof(*context), GFP_KERNEL);
 875         if (!context)
 876                 return -ENOMEM;
 877
 878         err = mlx4_qp_alloc(mdev->dev, qpn, qp);
 879         if (err) {
 880                 en_err(priv, "Failed to allocate qp #%x\n", qpn);
 881                 goto out;
 882         }
 883         qp->event = mlx4_en_sqp_event;
 884
 885         memset(context, 0, sizeof *context);
 886         mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
 887                                 qpn, ring->cqn, -1, context);
 888         context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
 889
 890         /* Cancel FCS removal if FW allows */
 891         if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
 892                 context->param3 |= cpu_to_be32(1 << 29);
 893                 ring->fcs_del = ETH_FCS_LEN;
 894         } else
 895                 ring->fcs_del = 0;
 896
 897         err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
 898         if (err) {
 899                 mlx4_qp_remove(mdev->dev, qp);
 900                 mlx4_qp_free(mdev->dev, qp);
 901         }
 902         mlx4_en_update_rx_prod_db(ring);
 903 out:
 904         kfree(context);
 905         return err;
 906 }
 907
 908 int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
 909 {
 910         int err;
 911         u32 qpn;
 912
 913         err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn);
 914         if (err) {
 915                 en_err(priv, "Failed reserving drop qpn\n");
 916                 return err;
 917         }
 918         err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp);
 919         if (err) {
 920                 en_err(priv, "Failed allocating drop qp\n");
 921                 mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
 922                 return err;
 923         }
 924
 925         return 0;
 926 }
 927
 928 void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
 929 {
 930         u32 qpn;
 931
 932         qpn = priv->drop_qp.qpn;
 933         mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
 934         mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
 935         mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
 936 }
 937
 938 /* Allocate rx qp's and configure them according to rss map */
 939 int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
 940 {
 941         struct mlx4_en_dev *mdev = priv->mdev;
 942         struct mlx4_en_rss_map *rss_map = &priv->rss_map;
 943         struct mlx4_qp_context context;
 944         struct mlx4_rss_context *rss_context;
 945         int rss_rings;
 946         void *ptr;
 947         u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
 948                         MLX4_RSS_TCP_IPV6);
 949         int i, qpn;
 950         int err = 0;
 951         int good_qps = 0;
 952         static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC,
 953                                 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD,
 954                                 0x593D56D9, 0xF3253C06, 0x2ADC1FFC};
 955
 956         en_dbg(DRV, priv, "Configuring rss steering\n");
 957         err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
 958                                     priv->rx_ring_num,
 959                                     &rss_map->base_qpn);
 960         if (err) {
 961                 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
 962                 return err;
 963         }
 964
 965         for (i = 0; i < priv->rx_ring_num; i++) {
 966                 qpn = rss_map->base_qpn + i;
 967                 err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
 968                                             &rss_map->state[i],
 969                                             &rss_map->qps[i]);
 970                 if (err)
 971                         goto rss_err;
 972
 973                 ++good_qps;
 974         }
 975
 976         /* Configure RSS indirection qp */
 977         err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
 978         if (err) {
 979                 en_err(priv, "Failed to allocate RSS indirection QP\n");
 980                 goto rss_err;
 981         }
 982         rss_map->indir_qp.event = mlx4_en_sqp_event;
 983         mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
 984                                 priv->rx_ring[0].cqn, -1, &context);
 985
 986         if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
 987                 rss_rings = priv->rx_ring_num;
 988         else
 989                 rss_rings = priv->prof->rss_rings;
 990
 991         ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
 992                                         + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
 993         rss_context = ptr;
 994         rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
 995                                             (rss_map->base_qpn));
 996         rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
 997         if (priv->mdev->profile.udp_rss) {
 998                 rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
 999                 rss_context->base_qpn_udp = rss_context->default_qpn;
1000         }
1001         rss_context->flags = rss_mask;
1002         rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1003         for (i = 0; i < 10; i++)
1004                 rss_context->rss_key[i] = cpu_to_be32(rsskey[i]);
1005
1006         err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
1007                                &rss_map->indir_qp, &rss_map->indir_state);
1008         if (err)
1009                 goto indir_err;
1010
1011         return 0;
1012
1013 indir_err:
1014         mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
1015                        MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
1016         mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
1017         mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
1018 rss_err:
1019         for (i = 0; i < good_qps; i++) {
1020                 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
1021                                MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
1022                 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
1023                 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
1024         }
1025         mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1026         return err;
1027 }
1028
1029 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
1030 {
1031         struct mlx4_en_dev *mdev = priv->mdev;
1032         struct mlx4_en_rss_map *rss_map = &priv->rss_map;
1033         int i;
1034
1035         mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
1036                        MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
1037         mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
1038         mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
1039
1040         for (i = 0; i < priv->rx_ring_num; i++) {
1041                 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
1042                                MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
1043                 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
1044                 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
1045         }
1046         mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1047 }