return value;
}
-static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring,
- u32 val)
-{
- rx_ring->next_to_use = val;
-
- /*
- * Force memory writes to complete before letting h/w
- * know there are new descriptors to fetch. (Only
- * applicable for weak-ordered memory model archs,
- * such as IA-64).
- */
- wmb();
- ixgbevf_write_tail(rx_ring, val);
-}
-
/**
* ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
* @adapter: pointer to adapter struct
return !!budget;
}
-/**
- * ixgbevf_receive_skb - Send a completed packet up the stack
- * @q_vector: structure containing interrupt and ring information
- * @skb: packet to send up
- * @status: hardware indication of status of receive
- * @rx_desc: rx descriptor
- **/
-static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
- struct sk_buff *skb, u8 status,
- union ixgbe_adv_rx_desc *rx_desc)
-{
- struct ixgbevf_adapter *adapter = q_vector->adapter;
- bool is_vlan = (status & IXGBE_RXD_STAT_VP);
- u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
-
- if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
- __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
-
- if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
- napi_gro_receive(&q_vector->napi, skb);
- else
- netif_rx(skb);
-}
-
/**
* ixgbevf_rx_skb - Helper function to determine proper Rx method
* @q_vector: structure containing interrupt and ring information
* @skb: packet to send up
- * @status: hardware indication of status of receive
- * @rx_desc: rx descriptor
**/
static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
- struct sk_buff *skb, u8 status,
- union ixgbe_adv_rx_desc *rx_desc)
+ struct sk_buff *skb)
{
#ifdef CONFIG_NET_RX_BUSY_POLL
skb_mark_napi_id(skb, &q_vector->napi);
return;
}
#endif /* CONFIG_NET_RX_BUSY_POLL */
-
- ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
+ if (!(q_vector->adapter->flags & IXGBE_FLAG_IN_NETPOLL))
+ napi_gro_receive(&q_vector->napi, skb);
+ else
+ netif_rx(skb);
}
-/**
- * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
- * @ring: pointer to Rx descriptor ring structure
- * @status_err: hardware indication of status of receive
+/* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
+ * @ring: structure containig ring specific data
+ * @rx_desc: current Rx descriptor being processed
* @skb: skb currently being received and modified
- **/
+ */
static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
- u32 status_err, struct sk_buff *skb)
+ union ixgbe_adv_rx_desc *rx_desc,
+ struct sk_buff *skb)
{
skb_checksum_none_assert(skb);
return;
/* if IP and error */
- if ((status_err & IXGBE_RXD_STAT_IPCS) &&
- (status_err & IXGBE_RXDADV_ERR_IPE)) {
+ if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
+ ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
ring->rx_stats.csum_err++;
return;
}
- if (!(status_err & IXGBE_RXD_STAT_L4CS))
+ if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
return;
- if (status_err & IXGBE_RXDADV_ERR_TCPE) {
+ if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
ring->rx_stats.csum_err++;
return;
}
skb->ip_summed = CHECKSUM_UNNECESSARY;
}
+/* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
+ * @rx_ring: rx descriptor ring packet is being transacted on
+ * @rx_desc: pointer to the EOP Rx descriptor
+ * @skb: pointer to current skb being populated
+ *
+ * This function checks the ring, descriptor, and packet information in
+ * order to populate the checksum, VLAN, protocol, and other fields within
+ * the skb.
+ */
+static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
+ union ixgbe_adv_rx_desc *rx_desc,
+ struct sk_buff *skb)
+{
+ ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
+
+ if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
+ u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
+ unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
+
+ if (test_bit(vid & VLAN_VID_MASK, active_vlans))
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
+ }
+
+ skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+}
+
+/**
+ * ixgbevf_is_non_eop - process handling of non-EOP buffers
+ * @rx_ring: Rx ring being processed
+ * @rx_desc: Rx descriptor for current buffer
+ * @skb: current socket buffer containing buffer in progress
+ *
+ * This function updates next to clean. If the buffer is an EOP buffer
+ * this function exits returning false, otherwise it will place the
+ * sk_buff in the next buffer to be chained and return true indicating
+ * that this is in fact a non-EOP buffer.
+ **/
+static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
+ union ixgbe_adv_rx_desc *rx_desc,
+ struct sk_buff *skb)
+{
+ u32 ntc = rx_ring->next_to_clean + 1;
+
+ /* fetch, update, and store next to clean */
+ ntc = (ntc < rx_ring->count) ? ntc : 0;
+ rx_ring->next_to_clean = ntc;
+
+ prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
+
+ if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
+ return false;
+
+ return true;
+}
+
+static bool ixgbevf_alloc_mapped_skb(struct ixgbevf_ring *rx_ring,
+ struct ixgbevf_rx_buffer *bi)
+{
+ struct sk_buff *skb = bi->skb;
+ dma_addr_t dma = bi->dma;
+
+ if (unlikely(skb))
+ return true;
+
+ skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
+ rx_ring->rx_buf_len);
+ if (unlikely(!skb)) {
+ rx_ring->rx_stats.alloc_rx_buff_failed++;
+ return false;
+ }
+
+ dma = dma_map_single(rx_ring->dev, skb->data,
+ rx_ring->rx_buf_len, DMA_FROM_DEVICE);
+
+ /* if mapping failed free memory back to system since
+ * there isn't much point in holding memory we can't use
+ */
+ if (dma_mapping_error(rx_ring->dev, dma)) {
+ dev_kfree_skb_any(skb);
+
+ rx_ring->rx_stats.alloc_rx_buff_failed++;
+ return false;
+ }
+
+ bi->skb = skb;
+ bi->dma = dma;
+
+ return true;
+}
+
/**
* ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
* @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
+ * @cleaned_count: number of buffers to replace
**/
static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
- int cleaned_count)
+ u16 cleaned_count)
{
union ixgbe_adv_rx_desc *rx_desc;
struct ixgbevf_rx_buffer *bi;
unsigned int i = rx_ring->next_to_use;
- while (cleaned_count--) {
- rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
- bi = &rx_ring->rx_buffer_info[i];
-
- if (!bi->skb) {
- struct sk_buff *skb;
+ /* nothing to do or no valid netdev defined */
+ if (!cleaned_count || !rx_ring->netdev)
+ return;
- skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
- rx_ring->rx_buf_len);
- if (!skb)
- goto no_buffers;
+ rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
+ bi = &rx_ring->rx_buffer_info[i];
+ i -= rx_ring->count;
- bi->skb = skb;
+ do {
+ if (!ixgbevf_alloc_mapped_skb(rx_ring, bi))
+ break;
- bi->dma = dma_map_single(rx_ring->dev, skb->data,
- rx_ring->rx_buf_len,
- DMA_FROM_DEVICE);
- if (dma_mapping_error(rx_ring->dev, bi->dma)) {
- dev_kfree_skb(skb);
- bi->skb = NULL;
- dev_err(rx_ring->dev, "Rx DMA map failed\n");
- break;
- }
- }
+ /* Refresh the desc even if pkt_addr didn't change
+ * because each write-back erases this info.
+ */
rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
+ rx_desc++;
+ bi++;
i++;
- if (i == rx_ring->count)
- i = 0;
- }
+ if (unlikely(!i)) {
+ rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
+ bi = rx_ring->rx_buffer_info;
+ i -= rx_ring->count;
+ }
+
+ /* clear the hdr_addr for the next_to_use descriptor */
+ rx_desc->read.hdr_addr = 0;
+
+ cleaned_count--;
+ } while (cleaned_count);
+
+ i += rx_ring->count;
-no_buffers:
- rx_ring->rx_stats.alloc_rx_buff_failed++;
- if (rx_ring->next_to_use != i)
- ixgbevf_release_rx_desc(rx_ring, i);
+ if (rx_ring->next_to_use != i) {
+ /* record the next descriptor to use */
+ rx_ring->next_to_use = i;
+
+ /* Force memory writes to complete before letting h/w
+ * know there are new descriptors to fetch. (Only
+ * applicable for weak-ordered memory model archs,
+ * such as IA-64).
+ */
+ wmb();
+ ixgbevf_write_tail(rx_ring, i);
+ }
}
static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *rx_ring,
int budget)
{
- union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
- struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
- struct sk_buff *skb;
- unsigned int i;
- u32 len, staterr;
- int cleaned_count = 0;
unsigned int total_rx_bytes = 0, total_rx_packets = 0;
+ u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
- i = rx_ring->next_to_clean;
- rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
- staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
- rx_buffer_info = &rx_ring->rx_buffer_info[i];
+ do {
+ union ixgbe_adv_rx_desc *rx_desc;
+ struct ixgbevf_rx_buffer *rx_buffer;
+ struct sk_buff *skb;
+ u16 ntc;
+
+ /* return some buffers to hardware, one at a time is too slow */
+ if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
+ ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
+ cleaned_count = 0;
+ }
- while (staterr & IXGBE_RXD_STAT_DD) {
- if (!budget)
+ ntc = rx_ring->next_to_clean;
+ rx_desc = IXGBEVF_RX_DESC(rx_ring, ntc);
+ rx_buffer = &rx_ring->rx_buffer_info[ntc];
+
+ if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
break;
- budget--;
- rmb(); /* read descriptor and rx_buffer_info after status DD */
- len = le16_to_cpu(rx_desc->wb.upper.length);
- skb = rx_buffer_info->skb;
- prefetch(skb->data - NET_IP_ALIGN);
- rx_buffer_info->skb = NULL;
+ /* This memory barrier is needed to keep us from reading
+ * any other fields out of the rx_desc until we know the
+ * RXD_STAT_DD bit is set
+ */
+ rmb();
- if (rx_buffer_info->dma) {
- dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
- rx_ring->rx_buf_len,
- DMA_FROM_DEVICE);
- rx_buffer_info->dma = 0;
- skb_put(skb, len);
- }
+ skb = rx_buffer->skb;
+ prefetch(skb->data);
- i++;
- if (i == rx_ring->count)
- i = 0;
+ /* pull the header of the skb in */
+ __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
- next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
- prefetch(next_rxd);
- cleaned_count++;
+ dma_unmap_single(rx_ring->dev, rx_buffer->dma,
+ rx_ring->rx_buf_len,
+ DMA_FROM_DEVICE);
- next_buffer = &rx_ring->rx_buffer_info[i];
+ /* clear skb reference in buffer info structure */
+ rx_buffer->skb = NULL;
+ rx_buffer->dma = 0;
- if (!(staterr & IXGBE_RXD_STAT_EOP)) {
- skb->next = next_buffer->skb;
- IXGBE_CB(skb->next)->prev = skb;
- rx_ring->rx_stats.non_eop_descs++;
- goto next_desc;
- }
+ cleaned_count++;
+
+ /* place incomplete frames back on ring for completion */
+ if (ixgbevf_is_non_eop(rx_ring, rx_desc, skb))
+ continue;
/* we should not be chaining buffers, if we did drop the skb */
if (IXGBE_CB(skb)->prev) {
skb = IXGBE_CB(skb)->prev;
dev_kfree_skb(this);
} while (skb);
- goto next_desc;
+ continue;
}
/* ERR_MASK will only have valid bits if EOP set */
- if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
+ if (unlikely(ixgbevf_test_staterr(rx_desc,
+ IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
dev_kfree_skb_irq(skb);
- goto next_desc;
+ continue;
}
- ixgbevf_rx_checksum(rx_ring, staterr, skb);
-
/* probably a little skewed due to removing CRC */
total_rx_bytes += skb->len;
total_rx_packets++;
- skb->protocol = eth_type_trans(skb, rx_ring->netdev);
-
/* Workaround hardware that can't do proper VEPA multicast
* source pruning.
*/
ether_addr_equal(rx_ring->netdev->dev_addr,
eth_hdr(skb)->h_source)) {
dev_kfree_skb_irq(skb);
- goto next_desc;
- }
-
- ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
-
-next_desc:
- rx_desc->wb.upper.status_error = 0;
-
- /* return some buffers to hardware, one at a time is too slow */
- if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
- ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
- cleaned_count = 0;
+ continue;
}
- /* use prefetched values */
- rx_desc = next_rxd;
- rx_buffer_info = &rx_ring->rx_buffer_info[i];
-
- staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
- }
+ /* populate checksum, VLAN, and protocol */
+ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
- rx_ring->next_to_clean = i;
- cleaned_count = ixgbevf_desc_unused(rx_ring);
+ ixgbevf_rx_skb(q_vector, skb);
- if (cleaned_count)
- ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
+ /* update budget accounting */
+ budget--;
+ } while (likely(budget));
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->stats.packets += total_rx_packets;
q_vector->rx.total_packets += total_rx_packets;
q_vector->rx.total_bytes += total_rx_bytes;
+ if (cleaned_count)
+ ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
+
return total_rx_packets;
}
projects / cascardo / linux.git / blobdiff