drivers/net/ethernet/sfc/rx.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2005-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/socket.h>
  12 #include <linux/in.h>
  13 #include <linux/slab.h>
  14 #include <linux/ip.h>
  15 #include <linux/tcp.h>
  16 #include <linux/udp.h>
  17 #include <linux/prefetch.h>
  18 #include <linux/moduleparam.h>
  19 #include <net/ip.h>
  20 #include <net/checksum.h>
  21 #include "net_driver.h"
  22 #include "efx.h"
  23 #include "nic.h"
  24 #include "selftest.h"
  25 #include "workarounds.h"
  26
  27 /* Number of RX descriptors pushed at once. */
  28 #define EFX_RX_BATCH  8
  29
  30 /* Maximum length for an RX descriptor sharing a page */
  31 #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state) \
  32                           - EFX_PAGE_IP_ALIGN)
  33
  34 /* Size of buffer allocated for skb header area. */
  35 #define EFX_SKB_HEADERS  64u
  36
  37 /* This is the percentage fill level below which new RX descriptors
  38  * will be added to the RX descriptor ring.
  39  */
  40 static unsigned int rx_refill_threshold;
  41
  42 /*
  43  * RX maximum head room required.
  44  *
  45  * This must be at least 1 to prevent overflow and at least 2 to allow
  46  * pipelined receives.
  47  */
  48 #define EFX_RXD_HEAD_ROOM 2
  49
  50 /* Offset of ethernet header within page */
  51 static inline unsigned int efx_rx_buf_offset(struct efx_nic *efx,
  52                                              struct efx_rx_buffer *buf)
  53 {
  54         return buf->page_offset + efx->type->rx_buffer_hash_size;
  55 }
  56
  57 static u8 *efx_rx_buf_eh(struct efx_nic *efx, struct efx_rx_buffer *buf)
  58 {
  59         return page_address(buf->page) + efx_rx_buf_offset(efx, buf);
  60 }
  61
  62 static inline u32 efx_rx_buf_hash(const u8 *eh)
  63 {
  64         /* The ethernet header is always directly after any hash. */
  65 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0
  66         return __le32_to_cpup((const __le32 *)(eh - 4));
  67 #else
  68         const u8 *data = eh - 4;
  69         return (u32)data[0]       |
  70                (u32)data[1] << 8  |
  71                (u32)data[2] << 16 |
  72                (u32)data[3] << 24;
  73 #endif
  74 }
  75
  76 /**
  77  * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
  78  *
  79  * @rx_queue:           Efx RX queue
  80  *
  81  * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
  82  * and populates struct efx_rx_buffers for each one. Return a negative error
  83  * code or 0 on success. If a single page can be split between two buffers,
  84  * then the page will either be inserted fully, or not at at all.
  85  */
  86 static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
  87 {
  88         struct efx_nic *efx = rx_queue->efx;
  89         struct efx_rx_buffer *rx_buf;
  90         struct page *page;
  91         unsigned int page_offset;
  92         struct efx_rx_page_state *state;
  93         dma_addr_t dma_addr;
  94         unsigned index, count;
  95
  96         /* We can split a page between two buffers */
  97         BUILD_BUG_ON(EFX_RX_BATCH & 1);
  98
  99         for (count = 0; count < EFX_RX_BATCH; ++count) {
 100                 page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC,
 101                                    efx->rx_buffer_order);
 102                 if (unlikely(page == NULL))
 103                         return -ENOMEM;
 104                 dma_addr = dma_map_page(&efx->pci_dev->dev, page, 0,
 105                                         PAGE_SIZE << efx->rx_buffer_order,
 106                                         DMA_FROM_DEVICE);
 107                 if (unlikely(dma_mapping_error(&efx->pci_dev->dev, dma_addr))) {
 108                         __free_pages(page, efx->rx_buffer_order);
 109                         return -EIO;
 110                 }
 111                 state = page_address(page);
 112                 state->refcnt = 0;
 113                 state->dma_addr = dma_addr;
 114
 115                 dma_addr += sizeof(struct efx_rx_page_state);
 116                 page_offset = sizeof(struct efx_rx_page_state);
 117
 118         split:
 119                 index = rx_queue->added_count & rx_queue->ptr_mask;
 120                 rx_buf = efx_rx_buffer(rx_queue, index);
 121                 rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN;
 122                 rx_buf->page = page;
 123                 rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN;
 124                 rx_buf->len = efx->rx_dma_len;
 125                 rx_buf->flags = 0;
 126                 ++rx_queue->added_count;
 127                 ++state->refcnt;
 128
 129                 if ((~count & 1) && (efx->rx_dma_len <= EFX_RX_HALF_PAGE)) {
 130                         /* Use the second half of the page */
 131                         get_page(page);
 132                         dma_addr += (PAGE_SIZE >> 1);
 133                         page_offset += (PAGE_SIZE >> 1);
 134                         ++count;
 135                         goto split;
 136                 }
 137         }
 138
 139         return 0;
 140 }
 141
 142 static void efx_unmap_rx_buffer(struct efx_nic *efx,
 143                                 struct efx_rx_buffer *rx_buf,
 144                                 unsigned int used_len)
 145 {
 146         if (rx_buf->page) {
 147                 struct efx_rx_page_state *state;
 148
 149                 state = page_address(rx_buf->page);
 150                 if (--state->refcnt == 0) {
 151                         dma_unmap_page(&efx->pci_dev->dev,
 152                                        state->dma_addr,
 153                                        PAGE_SIZE << efx->rx_buffer_order,
 154                                        DMA_FROM_DEVICE);
 155                 } else if (used_len) {
 156                         dma_sync_single_for_cpu(&efx->pci_dev->dev,
 157                                                 rx_buf->dma_addr, used_len,
 158                                                 DMA_FROM_DEVICE);
 159                 }
 160         }
 161 }
 162
 163 static void efx_free_rx_buffer(struct efx_nic *efx,
 164                                struct efx_rx_buffer *rx_buf)
 165 {
 166         if (rx_buf->page) {
 167                 __free_pages(rx_buf->page, efx->rx_buffer_order);
 168                 rx_buf->page = NULL;
 169         }
 170 }
 171
 172 static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
 173                                struct efx_rx_buffer *rx_buf)
 174 {
 175         efx_unmap_rx_buffer(rx_queue->efx, rx_buf, 0);
 176         efx_free_rx_buffer(rx_queue->efx, rx_buf);
 177 }
 178
 179 /* Attempt to resurrect the other receive buffer that used to share this page,
 180  * which had previously been passed up to the kernel and freed. */
 181 static void efx_resurrect_rx_buffer(struct efx_rx_queue *rx_queue,
 182                                     struct efx_rx_buffer *rx_buf)
 183 {
 184         struct efx_rx_page_state *state = page_address(rx_buf->page);
 185         struct efx_rx_buffer *new_buf;
 186         unsigned fill_level, index;
 187
 188         /* +1 because efx_rx_packet() incremented removed_count. +1 because
 189          * we'd like to insert an additional descriptor whilst leaving
 190          * EFX_RXD_HEAD_ROOM for the non-recycle path */
 191         fill_level = (rx_queue->added_count - rx_queue->removed_count + 2);
 192         if (unlikely(fill_level > rx_queue->max_fill)) {
 193                 /* We could place "state" on a list, and drain the list in
 194                  * efx_fast_push_rx_descriptors(). For now, this will do. */
 195                 return;
 196         }
 197
 198         ++state->refcnt;
 199         get_page(rx_buf->page);
 200
 201         index = rx_queue->added_count & rx_queue->ptr_mask;
 202         new_buf = efx_rx_buffer(rx_queue, index);
 203         new_buf->dma_addr = rx_buf->dma_addr ^ (PAGE_SIZE >> 1);
 204         new_buf->page = rx_buf->page;
 205         new_buf->len = rx_buf->len;
 206         ++rx_queue->added_count;
 207 }
 208
 209 /* Recycle the given rx buffer directly back into the rx_queue. There is
 210  * always room to add this buffer, because we've just popped a buffer. */
 211 static void efx_recycle_rx_buffer(struct efx_channel *channel,
 212                                   struct efx_rx_buffer *rx_buf)
 213 {
 214         struct efx_nic *efx = channel->efx;
 215         struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 216         struct efx_rx_buffer *new_buf;
 217         unsigned index;
 218
 219         rx_buf->flags = 0;
 220
 221         if (efx->rx_dma_len <= EFX_RX_HALF_PAGE &&
 222             page_count(rx_buf->page) == 1)
 223                 efx_resurrect_rx_buffer(rx_queue, rx_buf);
 224
 225         index = rx_queue->added_count & rx_queue->ptr_mask;
 226         new_buf = efx_rx_buffer(rx_queue, index);
 227
 228         memcpy(new_buf, rx_buf, sizeof(*new_buf));
 229         rx_buf->page = NULL;
 230         ++rx_queue->added_count;
 231 }
 232
 233 /**
 234  * efx_fast_push_rx_descriptors - push new RX descriptors quickly
 235  * @rx_queue:           RX descriptor queue
 236  *
 237  * This will aim to fill the RX descriptor queue up to
 238  * @rx_queue->@max_fill. If there is insufficient atomic
 239  * memory to do so, a slow fill will be scheduled.
 240  *
 241  * The caller must provide serialisation (none is used here). In practise,
 242  * this means this function must run from the NAPI handler, or be called
 243  * when NAPI is disabled.
 244  */
 245 void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
 246 {
 247         unsigned fill_level;
 248         int space, rc = 0;
 249
 250         /* Calculate current fill level, and exit if we don't need to fill */
 251         fill_level = (rx_queue->added_count - rx_queue->removed_count);
 252         EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);
 253         if (fill_level >= rx_queue->fast_fill_trigger)
 254                 goto out;
 255
 256         /* Record minimum fill level */
 257         if (unlikely(fill_level < rx_queue->min_fill)) {
 258                 if (fill_level)
 259                         rx_queue->min_fill = fill_level;
 260         }
 261
 262         space = rx_queue->max_fill - fill_level;
 263         EFX_BUG_ON_PARANOID(space < EFX_RX_BATCH);
 264
 265         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 266                    "RX queue %d fast-filling descriptor ring from"
 267                    " level %d to level %d\n",
 268                    efx_rx_queue_index(rx_queue), fill_level,
 269                    rx_queue->max_fill);
 270
 271
 272         do {
 273                 rc = efx_init_rx_buffers(rx_queue);
 274                 if (unlikely(rc)) {
 275                         /* Ensure that we don't leave the rx queue empty */
 276                         if (rx_queue->added_count == rx_queue->removed_count)
 277                                 efx_schedule_slow_fill(rx_queue);
 278                         goto out;
 279                 }
 280         } while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH);
 281
 282         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 283                    "RX queue %d fast-filled descriptor ring "
 284                    "to level %d\n", efx_rx_queue_index(rx_queue),
 285                    rx_queue->added_count - rx_queue->removed_count);
 286
 287  out:
 288         if (rx_queue->notified_count != rx_queue->added_count)
 289                 efx_nic_notify_rx_desc(rx_queue);
 290 }
 291
 292 void efx_rx_slow_fill(unsigned long context)
 293 {
 294         struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context;
 295
 296         /* Post an event to cause NAPI to run and refill the queue */
 297         efx_nic_generate_fill_event(rx_queue);
 298         ++rx_queue->slow_fill_count;
 299 }
 300
 301 static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue,
 302                                      struct efx_rx_buffer *rx_buf,
 303                                      int len)
 304 {
 305         struct efx_nic *efx = rx_queue->efx;
 306         unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;
 307
 308         if (likely(len <= max_len))
 309                 return;
 310
 311         /* The packet must be discarded, but this is only a fatal error
 312          * if the caller indicated it was
 313          */
 314         rx_buf->flags |= EFX_RX_PKT_DISCARD;
 315
 316         if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) {
 317                 if (net_ratelimit())
 318                         netif_err(efx, rx_err, efx->net_dev,
 319                                   " RX queue %d seriously overlength "
 320                                   "RX event (0x%x > 0x%x+0x%x). Leaking\n",
 321                                   efx_rx_queue_index(rx_queue), len, max_len,
 322                                   efx->type->rx_buffer_padding);
 323                 efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY);
 324         } else {
 325                 if (net_ratelimit())
 326                         netif_err(efx, rx_err, efx->net_dev,
 327                                   " RX queue %d overlength RX event "
 328                                   "(0x%x > 0x%x)\n",
 329                                   efx_rx_queue_index(rx_queue), len, max_len);
 330         }
 331
 332         efx_rx_queue_channel(rx_queue)->n_rx_overlength++;
 333 }
 334
 335 /* Pass a received packet up through GRO.  GRO can handle pages
 336  * regardless of checksum state and skbs with a good checksum.
 337  */
 338 static void efx_rx_packet_gro(struct efx_channel *channel,
 339                               struct efx_rx_buffer *rx_buf,
 340                               const u8 *eh)
 341 {
 342         struct napi_struct *napi = &channel->napi_str;
 343         gro_result_t gro_result;
 344         struct efx_nic *efx = channel->efx;
 345         struct page *page = rx_buf->page;
 346         struct sk_buff *skb;
 347
 348         rx_buf->page = NULL;
 349
 350         skb = napi_get_frags(napi);
 351         if (!skb) {
 352                 put_page(page);
 353                 return;
 354         }
 355
 356         if (efx->net_dev->features & NETIF_F_RXHASH)
 357                 skb->rxhash = efx_rx_buf_hash(eh);
 358
 359         skb_fill_page_desc(skb, 0, page,
 360                            efx_rx_buf_offset(efx, rx_buf), rx_buf->len);
 361
 362         skb->len = rx_buf->len;
 363         skb->data_len = rx_buf->len;
 364         skb->truesize += rx_buf->len;
 365         skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
 366                           CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
 367
 368         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 369
 370                 gro_result = napi_gro_frags(napi);
 371
 372         if (gro_result != GRO_DROP)
 373                 channel->irq_mod_score += 2;
 374 }
 375
 376 /* Allocate and construct an SKB around a struct page.*/
 377 static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
 378                                      struct efx_rx_buffer *rx_buf,
 379                                      u8 *eh, int hdr_len)
 380 {
 381         struct efx_nic *efx = channel->efx;
 382         struct sk_buff *skb;
 383
 384         /* Allocate an SKB to store the headers */
 385         skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN);
 386         if (unlikely(skb == NULL))
 387                 return NULL;
 388
 389         EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
 390
 391         skb_reserve(skb, EFX_PAGE_SKB_ALIGN);
 392
 393         skb->len = rx_buf->len;
 394         skb->truesize = rx_buf->len + sizeof(struct sk_buff);
 395         memcpy(skb->data, eh, hdr_len);
 396         skb->tail += hdr_len;
 397
 398         /* Append the remaining page onto the frag list */
 399         if (rx_buf->len > hdr_len) {
 400                 skb->data_len = skb->len - hdr_len;
 401                 skb_fill_page_desc(skb, 0, rx_buf->page,
 402                                    efx_rx_buf_offset(efx, rx_buf) + hdr_len,
 403                                    skb->data_len);
 404         } else {
 405                 __free_pages(rx_buf->page, efx->rx_buffer_order);
 406                 skb->data_len = 0;
 407         }
 408
 409         /* Ownership has transferred from the rx_buf to skb */
 410         rx_buf->page = NULL;
 411
 412         /* Move past the ethernet header */
 413         skb->protocol = eth_type_trans(skb, efx->net_dev);
 414
 415         return skb;
 416 }
 417
 418 void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
 419                    unsigned int len, u16 flags)
 420 {
 421         struct efx_nic *efx = rx_queue->efx;
 422         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 423         struct efx_rx_buffer *rx_buf;
 424
 425         rx_buf = efx_rx_buffer(rx_queue, index);
 426         rx_buf->flags |= flags;
 427
 428         /* This allows the refill path to post another buffer.
 429          * EFX_RXD_HEAD_ROOM ensures that the slot we are using
 430          * isn't overwritten yet.
 431          */
 432         rx_queue->removed_count++;
 433
 434         /* Validate the length encoded in the event vs the descriptor pushed */
 435         efx_rx_packet__check_len(rx_queue, rx_buf, len);
 436
 437         netif_vdbg(efx, rx_status, efx->net_dev,
 438                    "RX queue %d received id %x at %llx+%x %s%s\n",
 439                    efx_rx_queue_index(rx_queue), index,
 440                    (unsigned long long)rx_buf->dma_addr, len,
 441                    (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "",
 442                    (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : "");
 443
 444         /* Discard packet, if instructed to do so */
 445         if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) {
 446                 efx_recycle_rx_buffer(channel, rx_buf);
 447
 448                 /* Don't hold off the previous receive */
 449                 rx_buf = NULL;
 450                 goto out;
 451         }
 452
 453         /* Release and/or sync DMA mapping - assumes all RX buffers
 454          * consumed in-order per RX queue
 455          */
 456         efx_unmap_rx_buffer(efx, rx_buf, len);
 457
 458         /* Prefetch nice and early so data will (hopefully) be in cache by
 459          * the time we look at it.
 460          */
 461         prefetch(efx_rx_buf_eh(efx, rx_buf));
 462
 463         /* Pipeline receives so that we give time for packet headers to be
 464          * prefetched into cache.
 465          */
 466         rx_buf->len = len - efx->type->rx_buffer_hash_size;
 467 out:
 468         if (channel->rx_pkt)
 469                 __efx_rx_packet(channel, channel->rx_pkt);
 470         channel->rx_pkt = rx_buf;
 471 }
 472
 473 static void efx_rx_deliver(struct efx_channel *channel, u8 *eh,
 474                            struct efx_rx_buffer *rx_buf)
 475 {
 476         struct sk_buff *skb;
 477         u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS);
 478
 479         skb = efx_rx_mk_skb(channel, rx_buf, eh, hdr_len);
 480         if (unlikely(skb == NULL)) {
 481                 efx_free_rx_buffer(channel->efx, rx_buf);
 482                 return;
 483         }
 484         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 485
 486         /* Set the SKB flags */
 487         skb_checksum_none_assert(skb);
 488
 489         if (channel->type->receive_skb)
 490                 if (channel->type->receive_skb(channel, skb))
 491                         return;
 492
 493         /* Pass the packet up */
 494         netif_receive_skb(skb);
 495 }
 496
 497 /* Handle a received packet.  Second half: Touches packet payload. */
 498 void __efx_rx_packet(struct efx_channel *channel, struct efx_rx_buffer *rx_buf)
 499 {
 500         struct efx_nic *efx = channel->efx;
 501         u8 *eh = efx_rx_buf_eh(efx, rx_buf);
 502
 503         /* If we're in loopback test, then pass the packet directly to the
 504          * loopback layer, and free the rx_buf here
 505          */
 506         if (unlikely(efx->loopback_selftest)) {
 507                 efx_loopback_rx_packet(efx, eh, rx_buf->len);
 508                 efx_free_rx_buffer(efx, rx_buf);
 509                 return;
 510         }
 511
 512         if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
 513                 rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;
 514
 515         if (!channel->type->receive_skb)
 516                 efx_rx_packet_gro(channel, rx_buf, eh);
 517         else
 518                 efx_rx_deliver(channel, eh, rx_buf);
 519 }
 520
 521 int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
 522 {
 523         struct efx_nic *efx = rx_queue->efx;
 524         unsigned int entries;
 525         int rc;
 526
 527         /* Create the smallest power-of-two aligned ring */
 528         entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
 529         EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
 530         rx_queue->ptr_mask = entries - 1;
 531
 532         netif_dbg(efx, probe, efx->net_dev,
 533                   "creating RX queue %d size %#x mask %#x\n",
 534                   efx_rx_queue_index(rx_queue), efx->rxq_entries,
 535                   rx_queue->ptr_mask);
 536
 537         /* Allocate RX buffers */
 538         rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
 539                                    GFP_KERNEL);
 540         if (!rx_queue->buffer)
 541                 return -ENOMEM;
 542
 543         rc = efx_nic_probe_rx(rx_queue);
 544         if (rc) {
 545                 kfree(rx_queue->buffer);
 546                 rx_queue->buffer = NULL;
 547         }
 548         return rc;
 549 }
 550
 551 void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
 552 {
 553         struct efx_nic *efx = rx_queue->efx;
 554         unsigned int max_fill, trigger, max_trigger;
 555
 556         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 557                   "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
 558
 559         /* Initialise ptr fields */
 560         rx_queue->added_count = 0;
 561         rx_queue->notified_count = 0;
 562         rx_queue->removed_count = 0;
 563         rx_queue->min_fill = -1U;
 564
 565         /* Initialise limit fields */
 566         max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
 567         max_trigger = max_fill - EFX_RX_BATCH;
 568         if (rx_refill_threshold != 0) {
 569                 trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
 570                 if (trigger > max_trigger)
 571                         trigger = max_trigger;
 572         } else {
 573                 trigger = max_trigger;
 574         }
 575
 576         rx_queue->max_fill = max_fill;
 577         rx_queue->fast_fill_trigger = trigger;
 578
 579         /* Set up RX descriptor ring */
 580         rx_queue->enabled = true;
 581         efx_nic_init_rx(rx_queue);
 582 }
 583
 584 void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
 585 {
 586         int i;
 587         struct efx_rx_buffer *rx_buf;
 588
 589         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 590                   "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
 591
 592         /* A flush failure might have left rx_queue->enabled */
 593         rx_queue->enabled = false;
 594
 595         del_timer_sync(&rx_queue->slow_fill);
 596         efx_nic_fini_rx(rx_queue);
 597
 598         /* Release RX buffers NB start at index 0 not current HW ptr */
 599         if (rx_queue->buffer) {
 600                 for (i = 0; i <= rx_queue->ptr_mask; i++) {
 601                         rx_buf = efx_rx_buffer(rx_queue, i);
 602                         efx_fini_rx_buffer(rx_queue, rx_buf);
 603                 }
 604         }
 605 }
 606
 607 void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)
 608 {
 609         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 610                   "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
 611
 612         efx_nic_remove_rx(rx_queue);
 613
 614         kfree(rx_queue->buffer);
 615         rx_queue->buffer = NULL;
 616 }
 617
 618
 619 module_param(rx_refill_threshold, uint, 0444);
 620 MODULE_PARM_DESC(rx_refill_threshold,
 621                  "RX descriptor ring refill threshold (%)");
 622