#include "pvector.h"
#include "random.h"
#include "seq.h"
-#include "shash.h"
+#include "openvswitch/shash.h"
#include "sset.h"
#include "timeval.h"
#include "tnl-neigh-cache.h"
*
* dp_netdev_mutex (global)
* port_mutex
+ * non_pmd_mutex
*/
struct dp_netdev {
const struct dpif_class *const class;
/* Ports.
*
- * Protected by RCU. Take the mutex to add or remove ports. */
+ * Any lookup into 'ports' or any access to the dp_netdev_ports found
+ * through 'ports' requires taking 'port_mutex'. */
struct ovs_mutex port_mutex;
- struct cmap ports;
+ struct hmap ports;
struct seq *port_seq; /* Incremented whenever a port changes. */
/* Protects access to ofproto-dpif-upcall interface during revalidator
ovsthread_key_t per_pmd_key;
/* Cpu mask for pin of pmd threads. */
+ char *requested_pmd_cmask;
char *pmd_cmask;
+
uint64_t last_tnl_conf_seq;
};
static struct dp_netdev_port *dp_netdev_lookup_port(const struct dp_netdev *dp,
- odp_port_t);
+ odp_port_t)
+ OVS_REQUIRES(dp->port_mutex);
enum dp_stat_type {
DP_STAT_EXACT_HIT, /* Packets that had an exact match (emc). */
struct dp_netdev_port {
odp_port_t port_no;
struct netdev *netdev;
- struct cmap_node node; /* Node in dp_netdev's 'ports'. */
+ struct hmap_node node; /* Node in dp_netdev's 'ports'. */
struct netdev_saved_flags *sf;
unsigned n_rxq; /* Number of elements in 'rxq' */
struct netdev_rxq **rxq;
char *type; /* Port type as requested by user. */
- int latest_requested_n_rxq; /* Latest requested from netdev number
- of rx queues. */
};
/* Contained by struct dp_netdev_flow's 'stats' member. */
struct ovs_list node;
};
+/* Contained by struct dp_netdev_pmd_thread's 'port_cache' or 'tx_ports'. */
+struct tx_port {
+ odp_port_t port_no;
+ struct netdev *netdev;
+ struct hmap_node node;
+};
+
/* PMD: Poll modes drivers. PMD accesses devices via polling to eliminate
* the performance overhead of interrupt processing. Therefore netdev can
* not implement rx-wait for these devices. dpif-netdev needs to poll
/* Per thread exact-match cache. Note, the instance for cpu core
* NON_PMD_CORE_ID can be accessed by multiple threads, and thusly
- * need to be protected (e.g. by 'dp_netdev_mutex'). All other
- * instances will only be accessed by its own pmd thread. */
+ * need to be protected by 'non_pmd_mutex'. Every other instance
+ * will only be accessed by its own pmd thread. */
struct emc_cache flow_cache;
/* Classifier and Flow-Table.
struct latch exit_latch; /* For terminating the pmd thread. */
atomic_uint change_seq; /* For reloading pmd ports. */
pthread_t thread;
- int index; /* Idx of this pmd thread among pmd*/
- /* threads on same numa node. */
unsigned core_id; /* CPU core id of this pmd thread. */
int numa_id; /* numa node id of this pmd thread. */
- atomic_int tx_qid; /* Queue id used by this pmd thread to
- * send packets on all netdevs */
- struct ovs_mutex poll_mutex; /* Mutex for poll_list. */
+ /* Queue id used by this pmd thread to send packets on all netdevs.
+ * All tx_qid's are unique and less than 'ovs_numa_get_n_cores() + 1'. */
+ atomic_int tx_qid;
+
+ struct ovs_mutex port_mutex; /* Mutex for 'poll_list' and 'tx_ports'. */
/* List of rx queues to poll. */
struct ovs_list poll_list OVS_GUARDED;
- int poll_cnt; /* Number of elemints in poll_list. */
+ /* Number of elements in 'poll_list' */
+ int poll_cnt;
+ /* Map of 'tx_port's used for transmission. Written by the main thread,
+ * read by the pmd thread. */
+ struct hmap tx_ports OVS_GUARDED;
+
+ /* Map of 'tx_port' used in the fast path. This is a thread-local copy of
+ * 'tx_ports'. The instance for cpu core NON_PMD_CORE_ID can be accessed
+ * by multiple threads, and thusly need to be protected by 'non_pmd_mutex'.
+ * Every other instance will only be accessed by its own pmd thread. */
+ struct hmap port_cache;
/* Only a pmd thread can write on its own 'cycles' and 'stats'.
* The main thread keeps 'stats_zero' and 'cycles_zero' as base
};
static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no,
- struct dp_netdev_port **portp);
+ struct dp_netdev_port **portp)
+ OVS_REQUIRES(dp->port_mutex);
static int get_port_by_name(struct dp_netdev *dp, const char *devname,
- struct dp_netdev_port **portp);
+ struct dp_netdev_port **portp)
+ OVS_REQUIRES(dp->port_mutex);
static void dp_netdev_free(struct dp_netdev *)
OVS_REQUIRES(dp_netdev_mutex);
static int do_add_port(struct dp_netdev *dp, const char *devname,
static void dp_netdev_disable_upcall(struct dp_netdev *);
static void dp_netdev_pmd_reload_done(struct dp_netdev_pmd_thread *pmd);
static void dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd,
- struct dp_netdev *dp, int index,
- unsigned core_id, int numa_id);
+ struct dp_netdev *dp, unsigned core_id,
+ int numa_id);
static void dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread *pmd);
-static void dp_netdev_set_nonpmd(struct dp_netdev *dp);
+static void dp_netdev_set_nonpmd(struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex);
+
static struct dp_netdev_pmd_thread *dp_netdev_get_pmd(struct dp_netdev *dp,
unsigned core_id);
static struct dp_netdev_pmd_thread *
dp_netdev_pmd_get_next(struct dp_netdev *dp, struct cmap_position *pos);
static void dp_netdev_destroy_all_pmds(struct dp_netdev *dp);
static void dp_netdev_del_pmds_on_numa(struct dp_netdev *dp, int numa_id);
-static void dp_netdev_set_pmds_on_numa(struct dp_netdev *dp, int numa_id);
-static void dp_netdev_pmd_clear_poll_list(struct dp_netdev_pmd_thread *pmd);
-static void dp_netdev_del_port_from_pmd(struct dp_netdev_port *port,
- struct dp_netdev_pmd_thread *pmd);
+static void dp_netdev_set_pmds_on_numa(struct dp_netdev *dp, int numa_id)
+ OVS_REQUIRES(dp->port_mutex);
+static void dp_netdev_pmd_clear_ports(struct dp_netdev_pmd_thread *pmd);
static void dp_netdev_del_port_from_all_pmds(struct dp_netdev *dp,
struct dp_netdev_port *port);
-static void
-dp_netdev_add_port_to_pmds(struct dp_netdev *dp, struct dp_netdev_port *port);
-static void
-dp_netdev_add_rxq_to_pmd(struct dp_netdev_pmd_thread *pmd,
- struct dp_netdev_port *port, struct netdev_rxq *rx);
+static void dp_netdev_add_port_to_pmds(struct dp_netdev *dp,
+ struct dp_netdev_port *port);
+static void dp_netdev_add_port_tx_to_pmd(struct dp_netdev_pmd_thread *pmd,
+ struct dp_netdev_port *port);
+static void dp_netdev_add_rxq_to_pmd(struct dp_netdev_pmd_thread *pmd,
+ struct dp_netdev_port *port,
+ struct netdev_rxq *rx);
static struct dp_netdev_pmd_thread *
dp_netdev_less_loaded_pmd_on_numa(struct dp_netdev *dp, int numa_id);
-static void dp_netdev_reset_pmd_threads(struct dp_netdev *dp);
+static void dp_netdev_reset_pmd_threads(struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex);
static bool dp_netdev_pmd_try_ref(struct dp_netdev_pmd_thread *pmd);
static void dp_netdev_pmd_unref(struct dp_netdev_pmd_thread *pmd);
static void dp_netdev_pmd_flow_flush(struct dp_netdev_pmd_thread *pmd);
+static void pmd_load_cached_ports(struct dp_netdev_pmd_thread *pmd)
+ OVS_REQUIRES(pmd->port_mutex);
static inline bool emc_entry_alive(struct emc_entry *ce);
static void emc_clear_entry(struct emc_entry *ce);
ds_put_format(reply, "pmd thread numa_id %d core_id %u:\n",
pmd->numa_id, pmd->core_id);
- ovs_mutex_lock(&pmd->poll_mutex);
+ ovs_mutex_lock(&pmd->port_mutex);
LIST_FOR_EACH (poll, node, &pmd->poll_list) {
const char *name = netdev_get_name(poll->port->netdev);
ds_put_format(reply, " %d", netdev_rxq_get_queue_id(poll->rx));
prev_name = name;
}
- ovs_mutex_unlock(&pmd->poll_mutex);
+ ovs_mutex_unlock(&pmd->port_mutex);
ds_put_cstr(reply, "\n");
}
}
atomic_flag_clear(&dp->destroyed);
ovs_mutex_init(&dp->port_mutex);
- cmap_init(&dp->ports);
+ hmap_init(&dp->ports);
dp->port_seq = seq_create();
fat_rwlock_init(&dp->upcall_rwlock);
ovs_mutex_init_recursive(&dp->non_pmd_mutex);
ovsthread_key_create(&dp->per_pmd_key, NULL);
+ ovs_mutex_lock(&dp->port_mutex);
dp_netdev_set_nonpmd(dp);
- ovs_mutex_lock(&dp->port_mutex);
error = do_add_port(dp, name, "internal", ODPP_LOCAL);
ovs_mutex_unlock(&dp->port_mutex);
if (error) {
dp_netdev_free(struct dp_netdev *dp)
OVS_REQUIRES(dp_netdev_mutex)
{
- struct dp_netdev_port *port;
+ struct dp_netdev_port *port, *next;
shash_find_and_delete(&dp_netdevs, dp->name);
ovsthread_key_delete(dp->per_pmd_key);
ovs_mutex_lock(&dp->port_mutex);
- CMAP_FOR_EACH (port, node, &dp->ports) {
- /* PMD threads are destroyed here. do_del_port() cannot quiesce */
+ HMAP_FOR_EACH_SAFE (port, next, node, &dp->ports) {
do_del_port(dp, port);
}
ovs_mutex_unlock(&dp->port_mutex);
cmap_destroy(&dp->poll_threads);
seq_destroy(dp->port_seq);
- cmap_destroy(&dp->ports);
+ hmap_destroy(&dp->ports);
+ ovs_mutex_destroy(&dp->port_mutex);
/* Upcalls must be disabled at this point */
dp_netdev_destroy_upcall_lock(dp);
int old_seq;
if (pmd->core_id == NON_PMD_CORE_ID) {
+ ovs_mutex_lock(&pmd->dp->non_pmd_mutex);
+ ovs_mutex_lock(&pmd->port_mutex);
+ pmd_load_cached_ports(pmd);
+ ovs_mutex_unlock(&pmd->port_mutex);
+ ovs_mutex_unlock(&pmd->dp->non_pmd_mutex);
return;
}
}
static int
-do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
- odp_port_t port_no)
- OVS_REQUIRES(dp->port_mutex)
+port_create(const char *devname, const char *open_type, const char *type,
+ odp_port_t port_no, struct dp_netdev_port **portp)
{
struct netdev_saved_flags *sf;
struct dp_netdev_port *port;
- struct netdev *netdev;
enum netdev_flags flags;
- const char *open_type;
- int error = 0;
- int i, n_open_rxqs = 0;
+ struct netdev *netdev;
+ int n_open_rxqs = 0;
+ int i, error;
- /* Reject devices already in 'dp'. */
- if (!get_port_by_name(dp, devname, &port)) {
- error = EEXIST;
- goto out;
- }
+ *portp = NULL;
/* Open and validate network device. */
- open_type = dpif_netdev_port_open_type(dp->class, type);
error = netdev_open(devname, open_type, &netdev);
if (error) {
- goto out;
+ return error;
}
/* XXX reject non-Ethernet devices */
if (flags & NETDEV_LOOPBACK) {
VLOG_ERR("%s: cannot add a loopback device", devname);
error = EINVAL;
- goto out_close;
+ goto out;
}
if (netdev_is_pmd(netdev)) {
if (n_cores == OVS_CORE_UNSPEC) {
VLOG_ERR("%s, cannot get cpu core info", devname);
error = ENOENT;
- goto out_close;
+ goto out;
}
/* There can only be ovs_numa_get_n_cores() pmd threads,
* so creates a txq for each, and one extra for the non
* pmd threads. */
- error = netdev_set_multiq(netdev, n_cores + 1,
- netdev_requested_n_rxq(netdev));
+ error = netdev_set_tx_multiq(netdev, n_cores + 1);
if (error && (error != EOPNOTSUPP)) {
VLOG_ERR("%s, cannot set multiq", devname);
- goto out_close;
+ goto out;
+ }
+ }
+
+ if (netdev_is_reconf_required(netdev)) {
+ error = netdev_reconfigure(netdev);
+ if (error) {
+ goto out;
}
}
+
port = xzalloc(sizeof *port);
port->port_no = port_no;
port->netdev = netdev;
port->n_rxq = netdev_n_rxq(netdev);
- port->rxq = xmalloc(sizeof *port->rxq * port->n_rxq);
+ port->rxq = xcalloc(port->n_rxq, sizeof *port->rxq);
port->type = xstrdup(type);
- port->latest_requested_n_rxq = netdev_requested_n_rxq(netdev);
for (i = 0; i < port->n_rxq; i++) {
error = netdev_rxq_open(netdev, &port->rxq[i], i);
}
port->sf = sf;
- cmap_insert(&dp->ports, &port->node, hash_port_no(port_no));
-
- if (netdev_is_pmd(netdev)) {
- dp_netdev_add_port_to_pmds(dp, port);
- }
- seq_change(dp->port_seq);
+ *portp = port;
return 0;
free(port->type);
free(port->rxq);
free(port);
-out_close:
- netdev_close(netdev);
+
out:
+ netdev_close(netdev);
return error;
}
+static int
+do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
+ odp_port_t port_no)
+ OVS_REQUIRES(dp->port_mutex)
+{
+ struct dp_netdev_port *port;
+ int error;
+
+ /* Reject devices already in 'dp'. */
+ if (!get_port_by_name(dp, devname, &port)) {
+ return EEXIST;
+ }
+
+ error = port_create(devname, dpif_netdev_port_open_type(dp->class, type),
+ type, port_no, &port);
+ if (error) {
+ return error;
+ }
+
+ if (netdev_is_pmd(port->netdev)) {
+ int numa_id = netdev_get_numa_id(port->netdev);
+
+ ovs_assert(ovs_numa_numa_id_is_valid(numa_id));
+ dp_netdev_set_pmds_on_numa(dp, numa_id);
+ }
+
+ dp_netdev_add_port_to_pmds(dp, port);
+
+ hmap_insert(&dp->ports, &port->node, hash_port_no(port_no));
+ seq_change(dp->port_seq);
+
+ return 0;
+}
+
static int
dpif_netdev_port_add(struct dpif *dpif, struct netdev *netdev,
odp_port_t *port_nop)
static struct dp_netdev_port *
dp_netdev_lookup_port(const struct dp_netdev *dp, odp_port_t port_no)
+ OVS_REQUIRES(dp->port_mutex)
{
struct dp_netdev_port *port;
- CMAP_FOR_EACH_WITH_HASH (port, node, hash_port_no(port_no), &dp->ports) {
+ HMAP_FOR_EACH_WITH_HASH (port, node, hash_port_no(port_no), &dp->ports) {
if (port->port_no == port_no) {
return port;
}
static int
get_port_by_number(struct dp_netdev *dp,
odp_port_t port_no, struct dp_netdev_port **portp)
+ OVS_REQUIRES(dp->port_mutex)
{
if (!is_valid_port_number(port_no)) {
*portp = NULL;
{
struct dp_netdev_port *port;
- CMAP_FOR_EACH (port, node, &dp->ports) {
+ HMAP_FOR_EACH (port, node, &dp->ports) {
if (!strcmp(netdev_get_name(port->netdev), devname)) {
*portp = port;
return 0;
* is on numa node 'numa_id'. */
static bool
has_pmd_port_for_numa(struct dp_netdev *dp, int numa_id)
+ OVS_REQUIRES(dp->port_mutex)
{
struct dp_netdev_port *port;
- CMAP_FOR_EACH (port, node, &dp->ports) {
+ HMAP_FOR_EACH (port, node, &dp->ports) {
if (netdev_is_pmd(port->netdev)
&& netdev_get_numa_id(port->netdev) == numa_id) {
return true;
do_del_port(struct dp_netdev *dp, struct dp_netdev_port *port)
OVS_REQUIRES(dp->port_mutex)
{
- cmap_remove(&dp->ports, &port->node, hash_odp_port(port->port_no));
+ hmap_remove(&dp->ports, &port->node);
seq_change(dp->port_seq);
+
+ dp_netdev_del_port_from_all_pmds(dp, port);
+
if (netdev_is_pmd(port->netdev)) {
int numa_id = netdev_get_numa_id(port->netdev);
/* PMD threads can not be on invalid numa node. */
ovs_assert(ovs_numa_numa_id_is_valid(numa_id));
/* If there is no netdev on the numa node, deletes the pmd threads
- * for that numa. Else, deletes the queues from polling lists. */
+ * for that numa. */
if (!has_pmd_port_for_numa(dp, numa_id)) {
dp_netdev_del_pmds_on_numa(dp, numa_id);
- } else {
- dp_netdev_del_port_from_all_pmds(dp, port);
}
}
struct dp_netdev_port *port;
int error;
+ ovs_mutex_lock(&dp->port_mutex);
error = get_port_by_number(dp, port_no, &port);
if (!error && dpif_port) {
answer_port_query(port, dpif_port);
}
+ ovs_mutex_unlock(&dp->port_mutex);
return error;
}
}
struct dp_netdev_port_state {
- struct cmap_position position;
+ struct hmap_position position;
char *name;
};
{
struct dp_netdev_port_state *state = state_;
struct dp_netdev *dp = get_dp_netdev(dpif);
- struct cmap_node *node;
+ struct hmap_node *node;
int retval;
- node = cmap_next_position(&dp->ports, &state->position);
+ ovs_mutex_lock(&dp->port_mutex);
+ node = hmap_at_position(&dp->ports, &state->position);
if (node) {
struct dp_netdev_port *port;
} else {
retval = EOF;
}
+ ovs_mutex_unlock(&dp->port_mutex);
return retval;
}
/* Key */
offset = key_buf->size;
flow->key = ofpbuf_tail(key_buf);
- odp_parms.odp_in_port = netdev_flow->flow.in_port.odp_port;
odp_flow_key_from_flow(&odp_parms, key_buf);
flow->key_len = key_buf->size - offset;
/* Mask */
offset = mask_buf->size;
flow->mask = ofpbuf_tail(mask_buf);
- odp_parms.odp_in_port = wc.masks.in_port.odp_port;
odp_parms.key_buf = key_buf;
odp_flow_key_from_mask(&odp_parms, mask_buf);
flow->mask_len = mask_buf->size - offset;
struct dp_netdev *dp = get_dp_netdev(dpif);
struct dp_netdev_flow *netdev_flow;
struct dp_netdev_pmd_thread *pmd;
- unsigned pmd_id = get->pmd_id == PMD_ID_NULL
- ? NON_PMD_CORE_ID : get->pmd_id;
- int error = 0;
+ struct hmapx to_find = HMAPX_INITIALIZER(&to_find);
+ struct hmapx_node *node;
+ int error = EINVAL;
- pmd = dp_netdev_get_pmd(dp, pmd_id);
- if (!pmd) {
- return EINVAL;
+ if (get->pmd_id == PMD_ID_NULL) {
+ CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
+ if (dp_netdev_pmd_try_ref(pmd) && !hmapx_add(&to_find, pmd)) {
+ dp_netdev_pmd_unref(pmd);
+ }
+ }
+ } else {
+ pmd = dp_netdev_get_pmd(dp, get->pmd_id);
+ if (!pmd) {
+ goto out;
+ }
+ hmapx_add(&to_find, pmd);
}
- netdev_flow = dp_netdev_pmd_find_flow(pmd, get->ufid, get->key,
- get->key_len);
- if (netdev_flow) {
- dp_netdev_flow_to_dpif_flow(netdev_flow, get->buffer, get->buffer,
- get->flow, false);
- } else {
- error = ENOENT;
+ if (!hmapx_count(&to_find)) {
+ goto out;
}
- dp_netdev_pmd_unref(pmd);
+ HMAPX_FOR_EACH (node, &to_find) {
+ pmd = (struct dp_netdev_pmd_thread *) node->data;
+ netdev_flow = dp_netdev_pmd_find_flow(pmd, get->ufid, get->key,
+ get->key_len);
+ if (netdev_flow) {
+ dp_netdev_flow_to_dpif_flow(netdev_flow, get->buffer, get->buffer,
+ get->flow, false);
+ error = 0;
+ break;
+ } else {
+ error = ENOENT;
+ }
+ }
+ HMAPX_FOR_EACH (node, &to_find) {
+ pmd = (struct dp_netdev_pmd_thread *) node->data;
+ dp_netdev_pmd_unref(pmd);
+ }
+out:
+ hmapx_destroy(&to_find);
return error;
}
dp_netdev_flow_hash(&flow->ufid));
if (OVS_UNLIKELY(VLOG_IS_DBG_ENABLED())) {
- struct match match;
struct ds ds = DS_EMPTY_INITIALIZER;
+ struct ofpbuf key_buf, mask_buf;
+ struct odp_flow_key_parms odp_parms = {
+ .flow = &match->flow,
+ .mask = &match->wc.masks,
+ .support = dp_netdev_support,
+ };
+
+ ofpbuf_init(&key_buf, 0);
+ ofpbuf_init(&mask_buf, 0);
- match.tun_md.valid = false;
- match.flow = flow->flow;
- miniflow_expand(&flow->cr.mask->mf, &match.wc.masks);
+ odp_flow_key_from_flow(&odp_parms, &key_buf);
+ odp_parms.key_buf = &key_buf;
+ odp_flow_key_from_mask(&odp_parms, &mask_buf);
ds_put_cstr(&ds, "flow_add: ");
odp_format_ufid(ufid, &ds);
ds_put_cstr(&ds, " ");
- match_format(&match, &ds, OFP_DEFAULT_PRIORITY);
+ odp_flow_format(key_buf.data, key_buf.size,
+ mask_buf.data, mask_buf.size,
+ NULL, &ds, false);
ds_put_cstr(&ds, ", actions:");
format_odp_actions(&ds, actions, actions_len);
VLOG_DBG_RL(&upcall_rl, "%s", ds_cstr(&ds));
+ ofpbuf_uninit(&key_buf);
+ ofpbuf_uninit(&mask_buf);
ds_destroy(&ds);
}
* the 'non_pmd_mutex'. */
if (pmd->core_id == NON_PMD_CORE_ID) {
ovs_mutex_lock(&dp->non_pmd_mutex);
- ovs_mutex_lock(&dp->port_mutex);
}
/* The action processing expects the RSS hash to be valid, because
execute->actions_len);
if (pmd->core_id == NON_PMD_CORE_ID) {
- dp_netdev_pmd_unref(pmd);
- ovs_mutex_unlock(&dp->port_mutex);
ovs_mutex_unlock(&dp->non_pmd_mutex);
+ dp_netdev_pmd_unref(pmd);
}
return 0;
}
}
-/* Returns true if the configuration for rx queues or cpu mask
- * is changed. */
-static bool
-pmd_config_changed(const struct dp_netdev *dp, const char *cmask)
-{
- struct dp_netdev_port *port;
-
- CMAP_FOR_EACH (port, node, &dp->ports) {
- struct netdev *netdev = port->netdev;
- int requested_n_rxq = netdev_requested_n_rxq(netdev);
- if (netdev_is_pmd(netdev)
- && port->latest_requested_n_rxq != requested_n_rxq) {
- return true;
- }
- }
-
- if (dp->pmd_cmask != NULL && cmask != NULL) {
- return strcmp(dp->pmd_cmask, cmask);
- } else {
- return (dp->pmd_cmask != NULL || cmask != NULL);
- }
-}
-
-/* Resets pmd threads if the configuration for 'rxq's or cpu mask changes. */
+/* Changes the number or the affinity of pmd threads. The changes are actually
+ * applied in dpif_netdev_run(). */
static int
dpif_netdev_pmd_set(struct dpif *dpif, const char *cmask)
{
struct dp_netdev *dp = get_dp_netdev(dpif);
- if (pmd_config_changed(dp, cmask)) {
- struct dp_netdev_port *port;
-
- dp_netdev_destroy_all_pmds(dp);
-
- CMAP_FOR_EACH (port, node, &dp->ports) {
- struct netdev *netdev = port->netdev;
- int requested_n_rxq = netdev_requested_n_rxq(netdev);
- if (netdev_is_pmd(port->netdev)
- && port->latest_requested_n_rxq != requested_n_rxq) {
- int i, err;
-
- /* Closes the existing 'rxq's. */
- for (i = 0; i < netdev_n_rxq(port->netdev); i++) {
- netdev_rxq_close(port->rxq[i]);
- port->rxq[i] = NULL;
- }
- port->n_rxq = 0;
-
- /* Sets the new rx queue config. */
- err = netdev_set_multiq(port->netdev,
- ovs_numa_get_n_cores() + 1,
- requested_n_rxq);
- if (err && (err != EOPNOTSUPP)) {
- VLOG_ERR("Failed to set dpdk interface %s rx_queue to:"
- " %u", netdev_get_name(port->netdev),
- requested_n_rxq);
- return err;
- }
- port->latest_requested_n_rxq = requested_n_rxq;
- /* If the set_multiq() above succeeds, reopens the 'rxq's. */
- port->n_rxq = netdev_n_rxq(port->netdev);
- port->rxq = xrealloc(port->rxq, sizeof *port->rxq * port->n_rxq);
- for (i = 0; i < port->n_rxq; i++) {
- netdev_rxq_open(port->netdev, &port->rxq[i], i);
- }
- }
- }
- /* Reconfigures the cpu mask. */
- ovs_numa_set_cpu_mask(cmask);
- free(dp->pmd_cmask);
- dp->pmd_cmask = cmask ? xstrdup(cmask) : NULL;
-
- /* Restores the non-pmd. */
- dp_netdev_set_nonpmd(dp);
- /* Restores all pmd threads. */
- dp_netdev_reset_pmd_threads(dp);
+ if (!nullable_string_is_equal(dp->requested_pmd_cmask, cmask)) {
+ free(dp->requested_pmd_cmask);
+ dp->requested_pmd_cmask = nullable_xstrdup(cmask);
}
return 0;
}
}
+static int
+port_reconfigure(struct dp_netdev_port *port)
+{
+ struct netdev *netdev = port->netdev;
+ int i, err;
+
+ if (!netdev_is_reconf_required(netdev)) {
+ return 0;
+ }
+
+ /* Closes the existing 'rxq's. */
+ for (i = 0; i < port->n_rxq; i++) {
+ netdev_rxq_close(port->rxq[i]);
+ port->rxq[i] = NULL;
+ }
+ port->n_rxq = 0;
+
+ /* Allows 'netdev' to apply the pending configuration changes. */
+ err = netdev_reconfigure(netdev);
+ if (err && (err != EOPNOTSUPP)) {
+ VLOG_ERR("Failed to set interface %s new configuration",
+ netdev_get_name(netdev));
+ return err;
+ }
+ /* If the netdev_reconfigure() above succeeds, reopens the 'rxq's. */
+ port->rxq = xrealloc(port->rxq, sizeof *port->rxq * netdev_n_rxq(netdev));
+ for (i = 0; i < netdev_n_rxq(netdev); i++) {
+ err = netdev_rxq_open(netdev, &port->rxq[i], i);
+ if (err) {
+ return err;
+ }
+ port->n_rxq++;
+ }
+
+ return 0;
+}
+
+static void
+reconfigure_pmd_threads(struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex)
+{
+ struct dp_netdev_port *port, *next;
+
+ dp_netdev_destroy_all_pmds(dp);
+
+ HMAP_FOR_EACH_SAFE (port, next, node, &dp->ports) {
+ int err;
+
+ err = port_reconfigure(port);
+ if (err) {
+ hmap_remove(&dp->ports, &port->node);
+ seq_change(dp->port_seq);
+ port_destroy(port);
+ }
+ }
+ /* Reconfigures the cpu mask. */
+ ovs_numa_set_cpu_mask(dp->requested_pmd_cmask);
+ free(dp->pmd_cmask);
+ dp->pmd_cmask = nullable_xstrdup(dp->requested_pmd_cmask);
+
+ /* Restores the non-pmd. */
+ dp_netdev_set_nonpmd(dp);
+ /* Restores all pmd threads. */
+ dp_netdev_reset_pmd_threads(dp);
+}
+
+/* Returns true if one of the netdevs in 'dp' requires a reconfiguration */
+static bool
+ports_require_restart(const struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex)
+{
+ struct dp_netdev_port *port;
+
+ HMAP_FOR_EACH (port, node, &dp->ports) {
+ if (netdev_is_reconf_required(port->netdev)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
/* Return true if needs to revalidate datapath flows. */
static bool
dpif_netdev_run(struct dpif *dpif)
NON_PMD_CORE_ID);
uint64_t new_tnl_seq;
+ ovs_mutex_lock(&dp->port_mutex);
ovs_mutex_lock(&dp->non_pmd_mutex);
- CMAP_FOR_EACH (port, node, &dp->ports) {
+ HMAP_FOR_EACH (port, node, &dp->ports) {
if (!netdev_is_pmd(port->netdev)) {
int i;
}
}
ovs_mutex_unlock(&dp->non_pmd_mutex);
+
dp_netdev_pmd_unref(non_pmd);
+ if (!nullable_string_is_equal(dp->pmd_cmask, dp->requested_pmd_cmask)
+ || ports_require_restart(dp)) {
+ reconfigure_pmd_threads(dp);
+ }
+ ovs_mutex_unlock(&dp->port_mutex);
+
tnl_neigh_cache_run();
tnl_port_map_run();
new_tnl_seq = seq_read(tnl_conf_seq);
struct dp_netdev *dp = get_dp_netdev(dpif);
ovs_mutex_lock(&dp_netdev_mutex);
- CMAP_FOR_EACH (port, node, &dp->ports) {
+ ovs_mutex_lock(&dp->port_mutex);
+ HMAP_FOR_EACH (port, node, &dp->ports) {
+ netdev_wait_reconf_required(port->netdev);
if (!netdev_is_pmd(port->netdev)) {
int i;
}
}
}
+ ovs_mutex_unlock(&dp->port_mutex);
ovs_mutex_unlock(&dp_netdev_mutex);
seq_wait(tnl_conf_seq, dp->last_tnl_conf_seq);
}
+static void
+pmd_free_cached_ports(struct dp_netdev_pmd_thread *pmd)
+{
+ struct tx_port *tx_port_cached;
+
+ HMAP_FOR_EACH_POP (tx_port_cached, node, &pmd->port_cache) {
+ free(tx_port_cached);
+ }
+}
+
+/* Copies ports from 'pmd->tx_ports' (shared with the main thread) to
+ * 'pmd->port_cache' (thread local) */
+static void
+pmd_load_cached_ports(struct dp_netdev_pmd_thread *pmd)
+ OVS_REQUIRES(pmd->port_mutex)
+{
+ struct tx_port *tx_port, *tx_port_cached;
+
+ pmd_free_cached_ports(pmd);
+ hmap_shrink(&pmd->port_cache);
+
+ HMAP_FOR_EACH (tx_port, node, &pmd->tx_ports) {
+ tx_port_cached = xmemdup(tx_port, sizeof *tx_port_cached);
+ hmap_insert(&pmd->port_cache, &tx_port_cached->node,
+ hash_port_no(tx_port_cached->port_no));
+ }
+}
+
static int
-pmd_load_queues(struct dp_netdev_pmd_thread *pmd, struct rxq_poll **ppoll_list)
- OVS_REQUIRES(pmd->poll_mutex)
+pmd_load_queues_and_ports(struct dp_netdev_pmd_thread *pmd,
+ struct rxq_poll **ppoll_list)
{
struct rxq_poll *poll_list = *ppoll_list;
struct rxq_poll *poll;
int i;
+ ovs_mutex_lock(&pmd->port_mutex);
poll_list = xrealloc(poll_list, pmd->poll_cnt * sizeof *poll_list);
i = 0;
poll_list[i++] = *poll;
}
+ pmd_load_cached_ports(pmd);
+
+ ovs_mutex_unlock(&pmd->port_mutex);
+
*ppoll_list = poll_list;
- return pmd->poll_cnt;
+ return i;
}
static void *
unsigned int lc = 0;
struct rxq_poll *poll_list;
unsigned int port_seq = PMD_INITIAL_SEQ;
+ bool exiting;
int poll_cnt;
int i;
- poll_cnt = 0;
poll_list = NULL;
/* Stores the pmd thread's 'pmd' to 'per_pmd_key'. */
ovsthread_setspecific(pmd->dp->per_pmd_key, pmd);
- pmd_thread_setaffinity_cpu(pmd->core_id);
+ ovs_numa_thread_setaffinity_core(pmd->core_id);
+ dpdk_set_lcore_id(pmd->core_id);
+ poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list);
reload:
emc_cache_init(&pmd->flow_cache);
- ovs_mutex_lock(&pmd->poll_mutex);
- poll_cnt = pmd_load_queues(pmd, &poll_list);
- ovs_mutex_unlock(&pmd->poll_mutex);
-
/* List port/core affinity */
for (i = 0; i < poll_cnt; i++) {
VLOG_DBG("Core %d processing port \'%s\' with queue-id %d\n",
netdev_rxq_get_queue_id(poll_list[i].rx));
}
- /* Signal here to make sure the pmd finishes
- * reloading the updated configuration. */
- dp_netdev_pmd_reload_done(pmd);
-
for (;;) {
for (i = 0; i < poll_cnt; i++) {
dp_netdev_process_rxq_port(pmd, poll_list[i].port, poll_list[i].rx);
lc = 0;
- emc_cache_slow_sweep(&pmd->flow_cache);
coverage_try_clear();
- ovsrcu_quiesce();
+ if (!ovsrcu_try_quiesce()) {
+ emc_cache_slow_sweep(&pmd->flow_cache);
+ }
atomic_read_relaxed(&pmd->change_seq, &seq);
if (seq != port_seq) {
}
}
+ poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list);
+ exiting = latch_is_set(&pmd->exit_latch);
+ /* Signal here to make sure the pmd finishes
+ * reloading the updated configuration. */
+ dp_netdev_pmd_reload_done(pmd);
+
emc_cache_uninit(&pmd->flow_cache);
- if (!latch_is_set(&pmd->exit_latch)){
+ if (!exiting) {
goto reload;
}
- dp_netdev_pmd_reload_done(pmd);
-
free(poll_list);
+ pmd_free_cached_ports(pmd);
return NULL;
}
/* Sets the 'struct dp_netdev_pmd_thread' for non-pmd threads. */
static void
dp_netdev_set_nonpmd(struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex)
{
struct dp_netdev_pmd_thread *non_pmd;
+ struct dp_netdev_port *port;
non_pmd = xzalloc(sizeof *non_pmd);
- dp_netdev_configure_pmd(non_pmd, dp, 0, NON_PMD_CORE_ID,
- OVS_NUMA_UNSPEC);
+ dp_netdev_configure_pmd(non_pmd, dp, NON_PMD_CORE_ID, OVS_NUMA_UNSPEC);
+
+ HMAP_FOR_EACH (port, node, &dp->ports) {
+ dp_netdev_add_port_tx_to_pmd(non_pmd, port);
+ }
+
+ dp_netdev_reload_pmd__(non_pmd);
}
/* Caller must have valid pointer to 'pmd'. */
/* Configures the 'pmd' based on the input argument. */
static void
dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
- int index, unsigned core_id, int numa_id)
+ unsigned core_id, int numa_id)
{
pmd->dp = dp;
- pmd->index = index;
pmd->core_id = core_id;
pmd->numa_id = numa_id;
pmd->poll_cnt = 0;
xpthread_cond_init(&pmd->cond, NULL);
ovs_mutex_init(&pmd->cond_mutex);
ovs_mutex_init(&pmd->flow_mutex);
- ovs_mutex_init(&pmd->poll_mutex);
+ ovs_mutex_init(&pmd->port_mutex);
dpcls_init(&pmd->cls);
cmap_init(&pmd->flow_table);
ovs_list_init(&pmd->poll_list);
+ hmap_init(&pmd->tx_ports);
+ hmap_init(&pmd->port_cache);
/* init the 'flow_cache' since there is no
* actual thread created for NON_PMD_CORE_ID. */
if (core_id == NON_PMD_CORE_ID) {
{
dp_netdev_pmd_flow_flush(pmd);
dpcls_destroy(&pmd->cls);
+ hmap_destroy(&pmd->port_cache);
+ hmap_destroy(&pmd->tx_ports);
cmap_destroy(&pmd->flow_table);
ovs_mutex_destroy(&pmd->flow_mutex);
latch_destroy(&pmd->exit_latch);
xpthread_cond_destroy(&pmd->cond);
ovs_mutex_destroy(&pmd->cond_mutex);
- ovs_mutex_destroy(&pmd->poll_mutex);
+ ovs_mutex_destroy(&pmd->port_mutex);
free(pmd);
}
static void
dp_netdev_del_pmd(struct dp_netdev *dp, struct dp_netdev_pmd_thread *pmd)
{
- /* Uninit the 'flow_cache' since there is
- * no actual thread uninit it for NON_PMD_CORE_ID. */
+ /* NON_PMD_CORE_ID doesn't have a thread, so we don't have to synchronize,
+ * but extra cleanup is necessary */
if (pmd->core_id == NON_PMD_CORE_ID) {
emc_cache_uninit(&pmd->flow_cache);
+ pmd_free_cached_ports(pmd);
} else {
latch_set(&pmd->exit_latch);
dp_netdev_reload_pmd__(pmd);
xpthread_join(pmd->thread, NULL);
}
- /* Unref all ports and free poll_list. */
- dp_netdev_pmd_clear_poll_list(pmd);
+ dp_netdev_pmd_clear_ports(pmd);
/* Purges the 'pmd''s flows after stopping the thread, but before
* destroying the flows, so that the flow stats can be collected. */
free(free_idx);
}
-/* Deletes all rx queues from pmd->poll_list. */
+/* Deletes all rx queues from pmd->poll_list and all the ports from
+ * pmd->tx_ports. */
static void
-dp_netdev_pmd_clear_poll_list(struct dp_netdev_pmd_thread *pmd)
+dp_netdev_pmd_clear_ports(struct dp_netdev_pmd_thread *pmd)
{
struct rxq_poll *poll;
+ struct tx_port *port;
- ovs_mutex_lock(&pmd->poll_mutex);
+ ovs_mutex_lock(&pmd->port_mutex);
LIST_FOR_EACH_POP (poll, node, &pmd->poll_list) {
free(poll);
}
pmd->poll_cnt = 0;
- ovs_mutex_unlock(&pmd->poll_mutex);
+ HMAP_FOR_EACH_POP (port, node, &pmd->tx_ports) {
+ free(port);
+ }
+ ovs_mutex_unlock(&pmd->port_mutex);
}
-/* Deletes all rx queues of 'port' from poll_list of pmd thread and
- * reloads it if poll_list was changed. */
-static void
-dp_netdev_del_port_from_pmd(struct dp_netdev_port *port,
- struct dp_netdev_pmd_thread *pmd)
+static struct tx_port *
+tx_port_lookup(const struct hmap *hmap, odp_port_t port_no)
+{
+ struct tx_port *tx;
+
+ HMAP_FOR_EACH_IN_BUCKET (tx, node, hash_port_no(port_no), hmap) {
+ if (tx->port_no == port_no) {
+ return tx;
+ }
+ }
+
+ return NULL;
+}
+
+/* Deletes all rx queues of 'port' from 'poll_list', and the 'port' from
+ * 'tx_ports' of 'pmd' thread. Returns true if 'port' was found in 'pmd'
+ * (therefore a restart is required). */
+static bool
+dp_netdev_del_port_from_pmd__(struct dp_netdev_port *port,
+ struct dp_netdev_pmd_thread *pmd)
{
struct rxq_poll *poll, *next;
+ struct tx_port *tx;
bool found = false;
- ovs_mutex_lock(&pmd->poll_mutex);
+ ovs_mutex_lock(&pmd->port_mutex);
LIST_FOR_EACH_SAFE (poll, next, node, &pmd->poll_list) {
if (poll->port == port) {
found = true;
free(poll);
}
}
- ovs_mutex_unlock(&pmd->poll_mutex);
- if (found) {
- dp_netdev_reload_pmd__(pmd);
+
+ tx = tx_port_lookup(&pmd->tx_ports, port->port_no);
+ if (tx) {
+ hmap_remove(&pmd->tx_ports, &tx->node);
+ free(tx);
+ found = true;
+ }
+ ovs_mutex_unlock(&pmd->port_mutex);
+
+ return found;
+}
+
+/* Deletes 'port' from the 'poll_list' and from the 'tx_ports' of all the pmd
+ * threads. The pmd threads that need to be restarted are inserted in
+ * 'to_reload'. */
+static void
+dp_netdev_del_port_from_all_pmds__(struct dp_netdev *dp,
+ struct dp_netdev_port *port,
+ struct hmapx *to_reload)
+{
+ struct dp_netdev_pmd_thread *pmd;
+
+ CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
+ bool found;
+
+ found = dp_netdev_del_port_from_pmd__(port, pmd);
+
+ if (found) {
+ hmapx_add(to_reload, pmd);
+ }
}
}
-/* Deletes all rx queues of 'port' from all pmd threads of dp and
- * reloads them if needed. */
+/* Deletes 'port' from the 'poll_list' and from the 'tx_ports' of all the pmd
+ * threads. Reloads the threads if needed. */
static void
dp_netdev_del_port_from_all_pmds(struct dp_netdev *dp,
struct dp_netdev_port *port)
{
- int numa_id = netdev_get_numa_id(port->netdev);
struct dp_netdev_pmd_thread *pmd;
+ struct hmapx to_reload = HMAPX_INITIALIZER(&to_reload);
+ struct hmapx_node *node;
- CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
- if (pmd->numa_id == numa_id) {
- dp_netdev_del_port_from_pmd(port, pmd);
- }
+ dp_netdev_del_port_from_all_pmds__(dp, port, &to_reload);
+
+ HMAPX_FOR_EACH (node, &to_reload) {
+ pmd = (struct dp_netdev_pmd_thread *) node->data;
+ dp_netdev_reload_pmd__(pmd);
}
+
+ hmapx_destroy(&to_reload);
}
+
/* Returns PMD thread from this numa node with fewer rx queues to poll.
* Returns NULL if there is no PMD threads on this numa node.
* Can be called safely only by main thread. */
static void
dp_netdev_add_rxq_to_pmd(struct dp_netdev_pmd_thread *pmd,
struct dp_netdev_port *port, struct netdev_rxq *rx)
- OVS_REQUIRES(pmd->poll_mutex)
+ OVS_REQUIRES(pmd->port_mutex)
{
struct rxq_poll *poll = xmalloc(sizeof *poll);
pmd->poll_cnt++;
}
-/* Distributes all rx queues of 'port' between all PMD threads and reloads
- * them if needed. */
+/* Add 'port' to the tx port cache of 'pmd', which must be reloaded for the
+ * changes to take effect. */
static void
-dp_netdev_add_port_to_pmds(struct dp_netdev *dp, struct dp_netdev_port *port)
+dp_netdev_add_port_tx_to_pmd(struct dp_netdev_pmd_thread *pmd,
+ struct dp_netdev_port *port)
+{
+ struct tx_port *tx = xzalloc(sizeof *tx);
+
+ tx->netdev = port->netdev;
+ tx->port_no = port->port_no;
+
+ ovs_mutex_lock(&pmd->port_mutex);
+ hmap_insert(&pmd->tx_ports, &tx->node, hash_port_no(tx->port_no));
+ ovs_mutex_unlock(&pmd->port_mutex);
+}
+
+/* Distribute all rx queues of 'port' between PMD threads in 'dp'. The pmd
+ * threads that need to be restarted are inserted in 'to_reload'. */
+static void
+dp_netdev_add_port_rx_to_pmds(struct dp_netdev *dp,
+ struct dp_netdev_port *port,
+ struct hmapx *to_reload)
{
int numa_id = netdev_get_numa_id(port->netdev);
- struct dp_netdev_pmd_thread *pmd;
- struct hmapx to_reload;
- struct hmapx_node *node;
int i;
- hmapx_init(&to_reload);
- /* Cannot create pmd threads for invalid numa node. */
- ovs_assert(ovs_numa_numa_id_is_valid(numa_id));
+ if (!netdev_is_pmd(port->netdev)) {
+ return;
+ }
for (i = 0; i < port->n_rxq; i++) {
+ struct dp_netdev_pmd_thread *pmd;
+
pmd = dp_netdev_less_loaded_pmd_on_numa(dp, numa_id);
if (!pmd) {
- /* There is no pmd threads on this numa node. */
- dp_netdev_set_pmds_on_numa(dp, numa_id);
- /* Assigning of rx queues done. */
+ VLOG_WARN("There's no pmd thread on numa node %d", numa_id);
break;
}
- ovs_mutex_lock(&pmd->poll_mutex);
+ ovs_mutex_lock(&pmd->port_mutex);
dp_netdev_add_rxq_to_pmd(pmd, port, port->rxq[i]);
- ovs_mutex_unlock(&pmd->poll_mutex);
+ ovs_mutex_unlock(&pmd->port_mutex);
+
+ hmapx_add(to_reload, pmd);
+ }
+}
+
+/* Distributes all rx queues of 'port' between all PMD threads in 'dp' and
+ * inserts 'port' in the PMD threads 'tx_ports'. The pmd threads that need to
+ * be restarted are inserted in 'to_reload'. */
+static void
+dp_netdev_add_port_to_pmds__(struct dp_netdev *dp, struct dp_netdev_port *port,
+ struct hmapx *to_reload)
+{
+ struct dp_netdev_pmd_thread *pmd;
+
+ dp_netdev_add_port_rx_to_pmds(dp, port, to_reload);
- hmapx_add(&to_reload, pmd);
+ CMAP_FOR_EACH (pmd, node, &dp->poll_threads) {
+ dp_netdev_add_port_tx_to_pmd(pmd, port);
+ hmapx_add(to_reload, pmd);
}
+}
+
+/* Distributes all rx queues of 'port' between all PMD threads in 'dp', inserts
+ * 'port' in the PMD threads 'tx_ports' and reloads them, if needed. */
+static void
+dp_netdev_add_port_to_pmds(struct dp_netdev *dp, struct dp_netdev_port *port)
+{
+ struct dp_netdev_pmd_thread *pmd;
+ struct hmapx to_reload = HMAPX_INITIALIZER(&to_reload);
+ struct hmapx_node *node;
+
+ dp_netdev_add_port_to_pmds__(dp, port, &to_reload);
HMAPX_FOR_EACH (node, &to_reload) {
pmd = (struct dp_netdev_pmd_thread *) node->data;
hmapx_destroy(&to_reload);
}
-/* Checks the numa node id of 'netdev' and starts pmd threads for
- * the numa node. */
+/* Starts pmd threads for the numa node 'numa_id', if not already started.
+ * The function takes care of filling the threads tx port cache. */
static void
dp_netdev_set_pmds_on_numa(struct dp_netdev *dp, int numa_id)
+ OVS_REQUIRES(dp->port_mutex)
{
int n_pmds;
if (!ovs_numa_numa_id_is_valid(numa_id)) {
- VLOG_ERR("Cannot create pmd threads due to numa id (%d)"
- "invalid", numa_id);
- return ;
+ VLOG_WARN("Cannot create pmd threads due to numa id (%d) invalid",
+ numa_id);
+ return;
}
n_pmds = get_n_pmd_threads_on_numa(dp, numa_id);
* in which 'netdev' is on, do nothing. Else, creates the
* pmd threads for the numa node. */
if (!n_pmds) {
- int can_have, n_unpinned, i, index = 0;
- struct dp_netdev_pmd_thread **pmds;
- struct dp_netdev_port *port;
+ int can_have, n_unpinned, i;
n_unpinned = ovs_numa_get_n_unpinned_cores_on_numa(numa_id);
if (!n_unpinned) {
- VLOG_ERR("Cannot create pmd threads due to out of unpinned "
- "cores on numa node %d", numa_id);
+ VLOG_WARN("Cannot create pmd threads due to out of unpinned "
+ "cores on numa node %d", numa_id);
return;
}
/* If cpu mask is specified, uses all unpinned cores, otherwise
* tries creating NR_PMD_THREADS pmd threads. */
can_have = dp->pmd_cmask ? n_unpinned : MIN(n_unpinned, NR_PMD_THREADS);
- pmds = xzalloc(can_have * sizeof *pmds);
for (i = 0; i < can_have; i++) {
unsigned core_id = ovs_numa_get_unpinned_core_on_numa(numa_id);
- pmds[i] = xzalloc(sizeof **pmds);
- dp_netdev_configure_pmd(pmds[i], dp, i, core_id, numa_id);
- }
+ struct dp_netdev_pmd_thread *pmd = xzalloc(sizeof *pmd);
+ struct dp_netdev_port *port;
- /* Distributes rx queues of this numa node between new pmd threads. */
- CMAP_FOR_EACH (port, node, &dp->ports) {
- if (netdev_is_pmd(port->netdev)
- && netdev_get_numa_id(port->netdev) == numa_id) {
- for (i = 0; i < port->n_rxq; i++) {
- /* Make thread-safety analyser happy. */
- ovs_mutex_lock(&pmds[index]->poll_mutex);
- dp_netdev_add_rxq_to_pmd(pmds[index], port, port->rxq[i]);
- ovs_mutex_unlock(&pmds[index]->poll_mutex);
- index = (index + 1) % can_have;
- }
+ dp_netdev_configure_pmd(pmd, dp, core_id, numa_id);
+
+ HMAP_FOR_EACH (port, node, &dp->ports) {
+ dp_netdev_add_port_tx_to_pmd(pmd, port);
}
- }
- /* Actual start of pmd threads. */
- for (i = 0; i < can_have; i++) {
- pmds[i]->thread = ovs_thread_create("pmd", pmd_thread_main, pmds[i]);
+ pmd->thread = ovs_thread_create("pmd", pmd_thread_main, pmd);
}
- free(pmds);
VLOG_INFO("Created %d pmd threads on numa node %d", can_have, numa_id);
}
}
* new configuration. */
static void
dp_netdev_reset_pmd_threads(struct dp_netdev *dp)
+ OVS_REQUIRES(dp->port_mutex)
{
+ struct hmapx to_reload = HMAPX_INITIALIZER(&to_reload);
+ struct dp_netdev_pmd_thread *pmd;
struct dp_netdev_port *port;
+ struct hmapx_node *node;
- CMAP_FOR_EACH (port, node, &dp->ports) {
+ HMAP_FOR_EACH (port, node, &dp->ports) {
if (netdev_is_pmd(port->netdev)) {
int numa_id = netdev_get_numa_id(port->netdev);
dp_netdev_set_pmds_on_numa(dp, numa_id);
}
+ dp_netdev_add_port_rx_to_pmds(dp, port, &to_reload);
+ }
+
+ HMAPX_FOR_EACH (node, &to_reload) {
+ pmd = (struct dp_netdev_pmd_thread *) node->data;
+ dp_netdev_reload_pmd__(pmd);
}
+
+ hmapx_destroy(&to_reload);
}
static char *
struct odp_flow_key_parms odp_parms = {
.flow = flow,
.mask = &wc->masks,
- .odp_in_port = flow->in_port.odp_port,
.support = dp_netdev_support,
};
dp->upcall_cb = cb;
}
+static struct tx_port *
+pmd_tx_port_cache_lookup(const struct dp_netdev_pmd_thread *pmd,
+ odp_port_t port_no)
+{
+ return tx_port_lookup(&pmd->port_cache, port_no);
+}
+
static int
-push_tnl_action(const struct dp_netdev *dp,
+push_tnl_action(const struct dp_netdev_pmd_thread *pmd,
const struct nlattr *attr,
struct dp_packet_batch *batch)
{
- struct dp_netdev_port *tun_port;
+ struct tx_port *tun_port;
const struct ovs_action_push_tnl *data;
int err;
data = nl_attr_get(attr);
- tun_port = dp_netdev_lookup_port(dp, u32_to_odp(data->tnl_port));
+ tun_port = pmd_tx_port_cache_lookup(pmd, u32_to_odp(data->tnl_port));
if (!tun_port) {
err = -EINVAL;
goto error;
static void
dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
const struct nlattr *a, bool may_steal)
- OVS_NO_THREAD_SAFETY_ANALYSIS
{
struct dp_netdev_execute_aux *aux = aux_;
uint32_t *depth = recirc_depth_get();
struct dp_netdev_pmd_thread *pmd = aux->pmd;
struct dp_netdev *dp = pmd->dp;
int type = nl_attr_type(a);
- struct dp_netdev_port *p;
+ struct tx_port *p;
switch ((enum ovs_action_attr)type) {
case OVS_ACTION_ATTR_OUTPUT:
- p = dp_netdev_lookup_port(dp, u32_to_odp(nl_attr_get_u32(a)));
+ p = pmd_tx_port_cache_lookup(pmd, u32_to_odp(nl_attr_get_u32(a)));
if (OVS_LIKELY(p)) {
int tx_qid;
case OVS_ACTION_ATTR_TUNNEL_PUSH:
if (*depth < MAX_RECIRC_DEPTH) {
struct dp_packet_batch tnl_pkt;
+ struct dp_packet_batch *orig_packets_ = packets_;
int err;
if (!may_steal) {
dp_packet_batch_clone(&tnl_pkt, packets_);
packets_ = &tnl_pkt;
+ dp_packet_batch_reset_cutlen(orig_packets_);
}
- err = push_tnl_action(dp, a, packets_);
+ dp_packet_batch_apply_cutlen(packets_);
+
+ err = push_tnl_action(pmd, a, packets_);
if (!err) {
(*depth)++;
dp_netdev_recirculate(pmd, packets_);
case OVS_ACTION_ATTR_TUNNEL_POP:
if (*depth < MAX_RECIRC_DEPTH) {
+ struct dp_packet_batch *orig_packets_ = packets_;
odp_port_t portno = u32_to_odp(nl_attr_get_u32(a));
- p = dp_netdev_lookup_port(dp, portno);
+ p = pmd_tx_port_cache_lookup(pmd, portno);
if (p) {
struct dp_packet_batch tnl_pkt;
int i;
if (!may_steal) {
- dp_packet_batch_clone(&tnl_pkt, packets_);
- packets_ = &tnl_pkt;
+ dp_packet_batch_clone(&tnl_pkt, packets_);
+ packets_ = &tnl_pkt;
+ dp_packet_batch_reset_cutlen(orig_packets_);
}
+ dp_packet_batch_apply_cutlen(packets_);
+
netdev_pop_header(p->netdev, packets_);
if (!packets_->count) {
return;
case OVS_ACTION_ATTR_USERSPACE:
if (!fat_rwlock_tryrdlock(&dp->upcall_rwlock)) {
+ struct dp_packet_batch *orig_packets_ = packets_;
struct dp_packet **packets = packets_->packets;
const struct nlattr *userdata;
+ struct dp_packet_batch usr_pkt;
struct ofpbuf actions;
struct flow flow;
ovs_u128 ufid;
+ bool clone = false;
int i;
userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA);
ofpbuf_init(&actions, 0);
+ if (packets_->trunc) {
+ if (!may_steal) {
+ dp_packet_batch_clone(&usr_pkt, packets_);
+ packets_ = &usr_pkt;
+ packets = packets_->packets;
+ clone = true;
+ dp_packet_batch_reset_cutlen(orig_packets_);
+ }
+
+ dp_packet_batch_apply_cutlen(packets_);
+ }
+
for (i = 0; i < packets_->count; i++) {
flow_extract(packets[i], &flow);
dpif_flow_hash(dp->dpif, &flow, sizeof flow, &ufid);
dp_execute_userspace_action(pmd, packets[i], may_steal, &flow,
&ufid, &actions, userdata);
}
+
+ if (clone) {
+ dp_packet_delete_batch(packets_, true);
+ }
+
ofpbuf_uninit(&actions);
fat_rwlock_unlock(&dp->upcall_rwlock);
case OVS_ACTION_ATTR_SAMPLE:
case OVS_ACTION_ATTR_HASH:
case OVS_ACTION_ATTR_UNSPEC:
+ case OVS_ACTION_ATTR_TRUNC:
case __OVS_ACTION_ATTR_MAX:
OVS_NOT_REACHED();
}
dpif_dummy_change_port_number(struct unixctl_conn *conn, int argc OVS_UNUSED,
const char *argv[], void *aux OVS_UNUSED)
{
- struct dp_netdev_port *old_port;
- struct dp_netdev_port *new_port;
+ struct dp_netdev_port *port;
struct dp_netdev *dp;
odp_port_t port_no;
ovs_mutex_unlock(&dp_netdev_mutex);
ovs_mutex_lock(&dp->port_mutex);
- if (get_port_by_name(dp, argv[2], &old_port)) {
+ if (get_port_by_name(dp, argv[2], &port)) {
unixctl_command_reply_error(conn, "unknown port");
goto exit;
}
goto exit;
}
- /* Remove old port. */
- cmap_remove(&dp->ports, &old_port->node, hash_port_no(old_port->port_no));
- ovsrcu_postpone(free, old_port);
+ /* Remove port. */
+ hmap_remove(&dp->ports, &port->node);
+ dp_netdev_del_port_from_all_pmds(dp, port);
- /* Insert new port (cmap semantics mean we cannot re-insert 'old_port'). */
- new_port = xmemdup(old_port, sizeof *old_port);
- new_port->port_no = port_no;
- cmap_insert(&dp->ports, &new_port->node, hash_port_no(port_no));
+ /* Reinsert with new port number. */
+ port->port_no = port_no;
+ hmap_insert(&dp->ports, &port->node, hash_port_no(port_no));
+ dp_netdev_add_port_to_pmds(dp, port);
seq_change(dp->port_seq);
unixctl_command_reply(conn, NULL);
projects / cascardo / ovs.git / blobdiff