#include <unistd.h>
#include <stdio.h>
+#include "dp-packet.h"
#include "dpif-netdev.h"
#include "list.h"
#include "netdev-dpdk.h"
#include "ovs-numa.h"
#include "ovs-thread.h"
#include "ovs-rcu.h"
-#include "packet-dpif.h"
#include "packets.h"
#include "shash.h"
#include "sset.h"
#include "unaligned.h"
#include "timeval.h"
#include "unixctl.h"
-#include "vlog.h"
+#include "openvswitch/vlog.h"
VLOG_DEFINE_THIS_MODULE(dpdk);
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
#define MP_CACHE_SZ (256 * 2)
#define SOCKET0 0
-#define NON_PMD_THREAD_TX_QUEUE 0
-
#define NIC_PORT_RX_Q_SIZE 2048 /* Size of Physical NIC RX Queue, Max (n+32<=4096)*/
#define NIC_PORT_TX_Q_SIZE 2048 /* Size of Physical NIC TX Queue, Max (n+32<=4096)*/
static struct ovs_mutex dpdk_mutex = OVS_MUTEX_INITIALIZER;
/* Contains all 'struct dpdk_dev's. */
-static struct list dpdk_list OVS_GUARDED_BY(dpdk_mutex)
- = LIST_INITIALIZER(&dpdk_list);
+static struct ovs_list dpdk_list OVS_GUARDED_BY(dpdk_mutex)
+ = OVS_LIST_INITIALIZER(&dpdk_list);
-static struct list dpdk_mp_list OVS_GUARDED_BY(dpdk_mutex)
- = LIST_INITIALIZER(&dpdk_mp_list);
+static struct ovs_list dpdk_mp_list OVS_GUARDED_BY(dpdk_mutex)
+ = OVS_LIST_INITIALIZER(&dpdk_mp_list);
/* This mutex must be used by non pmd threads when allocating or freeing
* mbufs through mempools. Since dpdk_queue_pkts() and dpdk_queue_flush() may
int mtu;
int socket_id;
int refcount;
- struct list list_node OVS_GUARDED_BY(dpdk_mutex);
+ struct ovs_list list_node OVS_GUARDED_BY(dpdk_mutex);
};
/* There should be one 'struct dpdk_tx_queue' created for
so we have to keep them around once they've been created
*/
-static struct list dpdk_ring_list OVS_GUARDED_BY(dpdk_mutex)
- = LIST_INITIALIZER(&dpdk_ring_list);
+static struct ovs_list dpdk_ring_list OVS_GUARDED_BY(dpdk_mutex)
+ = OVS_LIST_INITIALIZER(&dpdk_ring_list);
struct dpdk_ring {
/* For the client rings */
struct rte_ring *cring_rx;
int user_port_id; /* User given port no, parsed from port name */
int eth_port_id; /* ethernet device port id */
- struct list list_node OVS_GUARDED_BY(dpdk_mutex);
+ struct ovs_list list_node OVS_GUARDED_BY(dpdk_mutex);
};
struct netdev_dpdk {
int link_reset_cnt;
/* In dpdk_list. */
- struct list list_node OVS_GUARDED_BY(dpdk_mutex);
+ struct ovs_list list_node OVS_GUARDED_BY(dpdk_mutex);
+ rte_spinlock_t dpdkr_tx_lock;
};
struct netdev_rxq_dpdk {
/* XXX this function should be called only by pmd threads (or by non pmd
* threads holding the nonpmd_mempool_mutex) */
void
-free_dpdk_buf(struct dpif_packet *p)
+free_dpdk_buf(struct dp_packet *p)
{
struct rte_mbuf *pkt = (struct rte_mbuf *) p;
unsigned i OVS_UNUSED)
{
struct rte_mbuf *m = _m;
- uint32_t buf_len = mp->elt_size - sizeof(struct dpif_packet);
+ uint32_t buf_len = mp->elt_size - sizeof(struct dp_packet);
- RTE_MBUF_ASSERT(mp->elt_size >= sizeof(struct dpif_packet));
+ RTE_MBUF_ASSERT(mp->elt_size >= sizeof(struct dp_packet));
memset(m, 0, mp->elt_size);
/* start of buffer is just after mbuf structure */
- m->buf_addr = (char *)m + sizeof(struct dpif_packet);
+ m->buf_addr = (char *)m + sizeof(struct dp_packet);
m->buf_physaddr = rte_mempool_virt2phy(mp, m) +
- sizeof(struct dpif_packet);
+ sizeof(struct dp_packet);
m->buf_len = (uint16_t)buf_len;
/* keep some headroom between start of buffer and data */
}
static void
-netdev_dpdk_set_txq(struct netdev_dpdk *netdev, unsigned int n_txqs)
+netdev_dpdk_alloc_txq(struct netdev_dpdk *netdev, unsigned int n_txqs)
{
int i;
/* Each index is considered as a cpu core id, since there should
* be one tx queue for each cpu core. */
for (i = 0; i < n_txqs; i++) {
- int core_id = ovs_numa_get_numa_id(i);
+ int numa_id = ovs_numa_get_numa_id(i);
/* If the corresponding core is not on the same numa node
* as 'netdev', flags the 'flush_tx'. */
- netdev->tx_q[i].flush_tx = netdev->socket_id == core_id;
+ netdev->tx_q[i].flush_tx = netdev->socket_id == numa_id;
}
}
OVS_REQUIRES(dpdk_mutex)
{
struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
+ int sid;
int err = 0;
ovs_mutex_init(&netdev->mutex);
ovs_mutex_lock(&netdev->mutex);
- netdev->socket_id = rte_eth_dev_socket_id(port_no);
- netdev_dpdk_set_txq(netdev, NR_QUEUE);
+ /* If the 'sid' is negative, it means that the kernel fails
+ * to obtain the pci numa info. In that situation, always
+ * use 'SOCKET0'. */
+ sid = rte_eth_dev_socket_id(port_no);
+ netdev->socket_id = sid < 0 ? SOCKET0 : sid;
+ netdev_dpdk_alloc_txq(netdev, NR_QUEUE);
netdev->port_id = port_no;
netdev->flags = 0;
netdev->mtu = ETHER_MTU;
netdev->max_packet_len = MTU_TO_MAX_LEN(netdev->mtu);
+ rte_spinlock_init(&netdev->dpdkr_tx_lock);
netdev->dpdk_mp = dpdk_mp_get(netdev->socket_id, netdev->mtu);
if (!netdev->dpdk_mp) {
return err;
}
+ ovs_mutex_lock(&dpdk_mutex);
ovs_mutex_lock(&netdev->mutex);
+
rte_eth_dev_stop(netdev->port_id);
+
netdev->up.n_txq = n_txq;
netdev->up.n_rxq = n_rxq;
+ rte_free(netdev->tx_q);
+ netdev_dpdk_alloc_txq(netdev, n_txq);
err = dpdk_eth_dev_init(netdev);
- if (!err && netdev->up.n_txq != n_txq) {
- rte_free(netdev->tx_q);
- netdev_dpdk_set_txq(netdev, n_txq);
- }
+
ovs_mutex_unlock(&netdev->mutex);
+ ovs_mutex_unlock(&dpdk_mutex);
return err;
}
}
static int
-netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct dpif_packet **packets,
+netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet **packets,
int *c)
{
struct netdev_rxq_dpdk *rx = netdev_rxq_dpdk_cast(rxq_);
/* Tx function. Transmit packets indefinitely */
static void
-dpdk_do_tx_copy(struct netdev *netdev, struct dpif_packet ** pkts, int cnt)
+dpdk_do_tx_copy(struct netdev *netdev, int qid, struct dp_packet ** pkts,
+ int cnt)
OVS_NO_THREAD_SAFETY_ANALYSIS
{
struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
ovs_mutex_unlock(&dev->mutex);
}
- dpdk_queue_pkts(dev, NON_PMD_THREAD_TX_QUEUE, mbufs, newcnt);
- dpdk_queue_flush(dev, NON_PMD_THREAD_TX_QUEUE);
+ dpdk_queue_pkts(dev, qid, mbufs, newcnt);
+ dpdk_queue_flush(dev, qid);
if (!thread_is_pmd()) {
ovs_mutex_unlock(&nonpmd_mempool_mutex);
}
}
-static int
-netdev_dpdk_send(struct netdev *netdev, int qid, struct dpif_packet **pkts,
- int cnt, bool may_steal)
+static inline void
+netdev_dpdk_send__(struct netdev_dpdk *dev, int qid,
+ struct dp_packet **pkts, int cnt, bool may_steal)
{
- struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
- int ret;
int i;
- if (!may_steal || pkts[0]->ofpbuf.source != OFPBUF_DPDK) {
- dpdk_do_tx_copy(netdev, pkts, cnt);
+ if (OVS_UNLIKELY(!may_steal ||
+ pkts[0]->ofpbuf.source != OFPBUF_DPDK)) {
+ struct netdev *netdev = &dev->up;
+
+ dpdk_do_tx_copy(netdev, qid, pkts, cnt);
if (may_steal) {
for (i = 0; i < cnt; i++) {
- dpif_packet_delete(pkts[i]);
+ dp_packet_delete(pkts[i]);
}
}
} else {
VLOG_WARN_RL(&rl, "Too big size %d max_packet_len %d",
(int)size , dev->max_packet_len);
- dpif_packet_delete(pkts[i]);
+ dp_packet_delete(pkts[i]);
dropped++;
next_tx_idx = i + 1;
}
ovs_mutex_unlock(&dev->mutex);
}
}
- ret = 0;
+}
- return ret;
+static int
+netdev_dpdk_eth_send(struct netdev *netdev, int qid,
+ struct dp_packet **pkts, int cnt, bool may_steal)
+{
+ struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
+
+ netdev_dpdk_send__(dev, qid, pkts, cnt, may_steal);
+ return 0;
}
static int
netdev_dpdk_set_miimon(struct netdev *netdev_ OVS_UNUSED,
long long int interval OVS_UNUSED)
{
- return 0;
+ return EOPNOTSUPP;
}
static int
return ENOMEM;
}
+ /* XXX: Add support for multiquque ring. */
err = snprintf(ring_name, 10, "%s_tx", dev_name);
if (err < 0) {
return -err;
}
- ivshmem->cring_tx = rte_ring_create(ring_name, DPDK_RING_SIZE, SOCKET0, 0);
+ /* Create single consumer/producer rings, netdev does explicit locking. */
+ ivshmem->cring_tx = rte_ring_create(ring_name, DPDK_RING_SIZE, SOCKET0,
+ RING_F_SP_ENQ | RING_F_SC_DEQ);
if (ivshmem->cring_tx == NULL) {
rte_free(ivshmem);
return ENOMEM;
return -err;
}
- ivshmem->cring_rx = rte_ring_create(ring_name, DPDK_RING_SIZE, SOCKET0, 0);
+ /* Create single consumer/producer rings, netdev does explicit locking. */
+ ivshmem->cring_rx = rte_ring_create(ring_name, DPDK_RING_SIZE, SOCKET0,
+ RING_F_SP_ENQ | RING_F_SC_DEQ);
if (ivshmem->cring_rx == NULL) {
rte_free(ivshmem);
return ENOMEM;
return dpdk_ring_create(dev_name, port_no, eth_port_id);
}
+static int
+netdev_dpdk_ring_send(struct netdev *netdev, int qid OVS_UNUSED,
+ struct dp_packet **pkts, int cnt, bool may_steal)
+{
+ struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
+
+ /* DPDK Rings have a single TX queue, Therefore needs locking. */
+ rte_spinlock_lock(&dev->dpdkr_tx_lock);
+ netdev_dpdk_send__(dev, 0, pkts, cnt, may_steal);
+ rte_spinlock_unlock(&dev->dpdkr_tx_lock);
+ return 0;
+}
+
static int
netdev_dpdk_ring_construct(struct netdev *netdev)
{
return err;
}
-#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, MULTIQ) \
+#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, MULTIQ, SEND) \
{ \
NAME, \
INIT, /* init */ \
netdev_dpdk_get_config, \
NULL, /* netdev_dpdk_set_config */ \
NULL, /* get_tunnel_config */ \
+ NULL, /* build header */ \
+ NULL, /* push header */ \
+ NULL, /* pop header */ \
netdev_dpdk_get_numa_id, /* get_numa_id */ \
MULTIQ, /* set_multiq */ \
\
- netdev_dpdk_send, /* send */ \
+ SEND, /* send */ \
NULL, /* send_wait */ \
\
netdev_dpdk_set_etheraddr, \
"dpdk",
dpdk_class_init,
netdev_dpdk_construct,
- netdev_dpdk_set_multiq);
+ netdev_dpdk_set_multiq,
+ netdev_dpdk_eth_send);
const struct netdev_class dpdk_ring_class =
NETDEV_DPDK_CLASS(
"dpdkr",
NULL,
netdev_dpdk_ring_construct,
- NULL);
+ NULL,
+ netdev_dpdk_ring_send);
void
netdev_dpdk_register(void)
VLOG_ERR("Thread affinity error %d",err);
return err;
}
- /* lcore_id 0 is reseved for use by non pmd threads. */
- ovs_assert(cpu);
+ /* NON_PMD_CORE_ID is reserved for use by non pmd threads. */
+ ovs_assert(cpu != NON_PMD_CORE_ID);
RTE_PER_LCORE(_lcore_id) = cpu;
return 0;
void
thread_set_nonpmd(void)
{
- /* We have to use 0 to allow non pmd threads to perform certain DPDK
- * operations, like rte_eth_dev_configure(). */
- RTE_PER_LCORE(_lcore_id) = 0;
+ /* We have to use NON_PMD_CORE_ID to allow non-pmd threads to perform
+ * certain DPDK operations, like rte_eth_dev_configure(). */
+ RTE_PER_LCORE(_lcore_id) = NON_PMD_CORE_ID;
}
static bool
thread_is_pmd(void)
{
- return rte_lcore_id() != 0;
+ return rte_lcore_id() != NON_PMD_CORE_ID;
}
projects / cascardo / ovs.git / blobdiff