X-Git-Url: http://git.cascardo.eti.br/?a=blobdiff_plain;f=lib%2Fdpif-netdev.c;h=a2868b134d3fe8556a5c2bde1a859e4d14040727;hb=61a2647e155c835e946407aa365c4d2adb5b57d5;hp=9ad9386916cff8ff1d313f1862944fd403be5e4c;hpb=cd527139bbe08729f18112eea5073ffd0f435979;p=cascardo%2Fovs.git diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 9ad938691..a2868b134 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -15,7 +15,7 @@ */ #include -#include "dpif.h" +#include "dpif-netdev.h" #include #include @@ -32,13 +32,15 @@ #include #include "classifier.h" +#include "cmap.h" #include "csum.h" #include "dpif.h" #include "dpif-provider.h" #include "dummy.h" #include "dynamic-string.h" +#include "fat-rwlock.h" #include "flow.h" -#include "hmap.h" +#include "cmap.h" #include "latch.h" #include "list.h" #include "meta-flow.h" @@ -51,6 +53,7 @@ #include "ofp-print.h" #include "ofpbuf.h" #include "ovs-rcu.h" +#include "packet-dpif.h" #include "packets.h" #include "poll-loop.h" #include "random.h" @@ -67,7 +70,7 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev); /* By default, choose a priority in the middle. */ #define NETDEV_RULE_PRIORITY 0x8000 -#define NR_THREADS 1 +#define FLOW_DUMP_MAX_BATCH 50 /* Use per thread recirc_depth to prevent recirculation loop. */ #define MAX_RECIRC_DEPTH 5 DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0) @@ -75,11 +78,6 @@ DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0) /* Configuration parameters. */ enum { MAX_FLOWS = 65536 }; /* Maximum number of flows in flow table. */ -/* Queues. */ -enum { MAX_QUEUE_LEN = 128 }; /* Maximum number of packets per queue. */ -enum { QUEUE_MASK = MAX_QUEUE_LEN - 1 }; -BUILD_ASSERT_DECL(IS_POW2(MAX_QUEUE_LEN)); - /* Protects against changes to 'dp_netdevs'. */ static struct ovs_mutex dp_netdev_mutex = OVS_MUTEX_INITIALIZER; @@ -87,26 +85,7 @@ static struct ovs_mutex dp_netdev_mutex = OVS_MUTEX_INITIALIZER; static struct shash dp_netdevs OVS_GUARDED_BY(dp_netdev_mutex) = SHASH_INITIALIZER(&dp_netdevs); -struct dp_netdev_upcall { - struct dpif_upcall upcall; /* Queued upcall information. */ - struct ofpbuf buf; /* ofpbuf instance for upcall.packet. */ -}; - -/* A queue passing packets from a struct dp_netdev to its clients (handlers). - * - * - * Thread-safety - * ============= - * - * Any access at all requires the owning 'dp_netdev''s queue_rwlock and - * its own mutex. */ -struct dp_netdev_queue { - struct ovs_mutex mutex; - struct seq *seq; /* Incremented whenever a packet is queued. */ - struct dp_netdev_upcall upcalls[MAX_QUEUE_LEN] OVS_GUARDED; - unsigned int head OVS_GUARDED; - unsigned int tail OVS_GUARDED; -}; +static struct vlog_rate_limit upcall_rl = VLOG_RATE_LIMIT_INIT(600, 600); /* Datapath based on the network device interface from netdev.h. * @@ -120,37 +99,24 @@ struct dp_netdev_queue { * Acquisition order is, from outermost to innermost: * * dp_netdev_mutex (global) - * port_rwlock + * port_mutex * flow_mutex - * cls.rwlock - * queue_rwlock */ struct dp_netdev { const struct dpif_class *const class; const char *const name; + struct dpif *dpif; struct ovs_refcount ref_cnt; atomic_flag destroyed; /* Flows. * - * Readers of 'cls' and 'flow_table' must take a 'cls->rwlock' read lock. - * - * Writers of 'cls' and 'flow_table' must take the 'flow_mutex' and then - * the 'cls->rwlock' write lock. (The outer 'flow_mutex' allows writers to - * atomically perform multiple operations on 'cls' and 'flow_table'.) + * Writers of 'flow_table' must take the 'flow_mutex'. Corresponding + * changes to 'cls' must be made while still holding the 'flow_mutex'. */ struct ovs_mutex flow_mutex; - struct classifier cls; /* Classifier. Protected by cls.rwlock. */ - struct hmap flow_table OVS_GUARDED; /* Flow table. */ - - /* Queues. - * - * 'queue_rwlock' protects the modification of 'handler_queues' and - * 'n_handlers'. The queue elements are protected by its - * 'handler_queues''s mutex. */ - struct fat_rwlock queue_rwlock; - struct dp_netdev_queue *handler_queues; - uint32_t n_handlers; + struct classifier cls; + struct cmap flow_table OVS_GUARDED; /* Flow table. */ /* Statistics. * @@ -159,12 +125,17 @@ struct dp_netdev { /* Ports. * - * Any lookup into 'ports' or any access to the dp_netdev_ports found - * through 'ports' requires taking 'port_rwlock'. */ - struct ovs_rwlock port_rwlock; - struct hmap ports OVS_GUARDED; + * Protected by RCU. Take the mutex to add or remove ports. */ + struct ovs_mutex port_mutex; + struct cmap ports; struct seq *port_seq; /* Incremented whenever a port changes. */ + /* Protects access to ofproto-dpif-upcall interface during revalidator + * thread synchronization. */ + struct fat_rwlock upcall_rwlock; + upcall_callback *upcall_cb; /* Callback function for executing upcalls. */ + void *upcall_aux; + /* Forwarding threads. */ struct latch exit_latch; struct pmd_thread *pmd_threads; @@ -173,8 +144,7 @@ struct dp_netdev { }; static struct dp_netdev_port *dp_netdev_lookup_port(const struct dp_netdev *dp, - odp_port_t) - OVS_REQ_RDLOCK(dp->port_rwlock); + odp_port_t); enum dp_stat_type { DP_STAT_HIT, /* Packets that matched in the flow table. */ @@ -194,7 +164,7 @@ struct dp_netdev_stats { /* A port in a netdev-based datapath. */ struct dp_netdev_port { - struct hmap_node node; /* Node in dp_netdev's 'ports'. */ + struct cmap_node node; /* Node in dp_netdev's 'ports'. */ odp_port_t port_no; struct netdev *netdev; struct netdev_saved_flags *sf; @@ -203,6 +173,20 @@ struct dp_netdev_port { char *type; /* Port type as requested by user. */ }; + +/* Stores a miniflow */ + +/* There are fields in the flow structure that we never use. Therefore we can + * save a few words of memory */ +#define NETDEV_KEY_BUF_SIZE_U32 (FLOW_U32S - MINI_N_INLINE \ + - FLOW_U32_SIZE(regs) \ + - FLOW_U32_SIZE(metadata) \ + ) +struct netdev_flow_key { + struct miniflow flow; + uint32_t buf[NETDEV_KEY_BUF_SIZE_U32]; +}; + /* A flow in dp_netdev's 'flow_table'. * * @@ -229,19 +213,13 @@ struct dp_netdev_port { * Rules * ----- * - * A flow 'flow' may be accessed without a risk of being freed by code that - * holds a read-lock or write-lock on 'cls->rwlock' or that owns a reference to - * 'flow->ref_cnt' (or both). Code that needs to hold onto a flow for a while - * should take 'cls->rwlock', find the flow it needs, increment 'flow->ref_cnt' - * with dpif_netdev_flow_ref(), and drop 'cls->rwlock'. + * A flow 'flow' may be accessed without a risk of being freed during an RCU + * grace period. Code that needs to hold onto a flow for a while + * should try incrementing 'flow->ref_cnt' with dp_netdev_flow_ref(). * * 'flow->ref_cnt' protects 'flow' from being freed. It doesn't protect the - * flow from being deleted from 'cls' (that's 'cls->rwlock') and it doesn't - * protect members of 'flow' from modification (that's 'flow->mutex'). - * - * 'flow->mutex' protects the members of 'flow' from modification. It doesn't - * protect the flow from being deleted from 'cls' (that's 'cls->rwlock') and it - * doesn't prevent the flow from being freed (that's 'flow->ref_cnt'). + * flow from being deleted from 'cls' and it doesn't protect members of 'flow' + * from modification. * * Some members, marked 'const', are immutable. Accessing other members * requires synchronization, as noted in more detail below. @@ -251,28 +229,25 @@ struct dp_netdev_flow { const struct cls_rule cr; /* In owning dp_netdev's 'cls'. */ /* Hash table index by unmasked flow. */ - const struct hmap_node node; /* In owning dp_netdev's 'flow_table'. */ + const struct cmap_node node; /* In owning dp_netdev's 'flow_table'. */ const struct flow flow; /* The flow that created this entry. */ - /* Protects members marked OVS_GUARDED. - * - * Acquire after datapath's flow_mutex. */ - struct ovs_mutex mutex OVS_ACQ_AFTER(dp_netdev_mutex); + /* Number of references. + * The classifier owns one reference. + * Any thread trying to keep a rule from being freed should hold its own + * reference. */ + struct ovs_refcount ref_cnt; /* Statistics. * * Reading or writing these members requires 'mutex'. */ struct ovsthread_stats stats; /* Contains "struct dp_netdev_flow_stats". */ - /* Actions. - * - * Reading 'actions' requires 'mutex'. - * Writing 'actions' requires 'mutex' and (to allow for transactions) the - * datapath's flow_mutex. */ + /* Actions. */ OVSRCU_TYPE(struct dp_netdev_actions *) actions; }; -static void dp_netdev_flow_free(struct dp_netdev_flow *); +static void dp_netdev_flow_unref(struct dp_netdev_flow *); /* Contained by struct dp_netdev_flow's 'stats' member. */ struct dp_netdev_flow_stats { @@ -290,10 +265,7 @@ struct dp_netdev_flow_stats { * Thread-safety * ============= * - * A struct dp_netdev_actions 'actions' may be accessed without a risk of being - * freed by code that holds a read-lock or write-lock on 'flow->mutex' (where - * 'flow' is the dp_netdev_flow for which 'flow->actions == actions') or that - * owns a reference to 'actions->ref_cnt' (or both). */ + * A struct dp_netdev_actions 'actions' is protected with RCU. */ struct dp_netdev_actions { /* These members are immutable: they do not change during the struct's * lifetime. */ @@ -323,9 +295,10 @@ struct pmd_thread { pthread_t thread; int id; atomic_uint change_seq; - char *name; }; +#define PMD_INITIAL_SEQ 1 + /* Interface to netdev-based datapath. */ struct dpif_netdev { struct dpif dpif; @@ -334,36 +307,30 @@ struct dpif_netdev { }; static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no, - struct dp_netdev_port **portp) - OVS_REQ_RDLOCK(dp->port_rwlock); + struct dp_netdev_port **portp); static int get_port_by_name(struct dp_netdev *dp, const char *devname, - struct dp_netdev_port **portp) - OVS_REQ_RDLOCK(dp->port_rwlock); + struct dp_netdev_port **portp); static void dp_netdev_free(struct dp_netdev *) OVS_REQUIRES(dp_netdev_mutex); static void dp_netdev_flow_flush(struct dp_netdev *); static int do_add_port(struct dp_netdev *dp, const char *devname, const char *type, odp_port_t port_no) - OVS_REQ_WRLOCK(dp->port_rwlock); -static int do_del_port(struct dp_netdev *dp, odp_port_t port_no) - OVS_REQ_WRLOCK(dp->port_rwlock); -static void dp_netdev_destroy_all_queues(struct dp_netdev *dp) - OVS_REQ_WRLOCK(dp->queue_rwlock); + OVS_REQUIRES(dp->port_mutex); +static void do_del_port(struct dp_netdev *dp, struct dp_netdev_port *) + OVS_REQUIRES(dp->port_mutex); static int dpif_netdev_open(const struct dpif_class *, const char *name, bool create, struct dpif **); -static int dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *, - int queue_no, int type, - const struct flow *, - const struct nlattr *userdata); static void dp_netdev_execute_actions(struct dp_netdev *dp, - const struct flow *, struct ofpbuf *, bool may_steal, - struct pkt_metadata *, + struct dpif_packet **, int c, + bool may_steal, struct pkt_metadata *, const struct nlattr *actions, size_t actions_len); -static void dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet, - struct pkt_metadata *); +static void dp_netdev_port_input(struct dp_netdev *dp, + struct dpif_packet **packets, int cnt, + odp_port_t port_no); static void dp_netdev_set_pmd_threads(struct dp_netdev *, int n); +static void dp_netdev_disable_upcall(struct dp_netdev *); static struct dpif_netdev * dpif_netdev_cast(const struct dpif *dpif) @@ -379,12 +346,19 @@ get_dp_netdev(const struct dpif *dpif) } static int -dpif_netdev_enumerate(struct sset *all_dps) +dpif_netdev_enumerate(struct sset *all_dps, + const struct dpif_class *dpif_class) { struct shash_node *node; ovs_mutex_lock(&dp_netdev_mutex); SHASH_FOR_EACH(node, &dp_netdevs) { + struct dp_netdev *dp = node->data; + if (dpif_class != dp->class) { + /* 'dp_netdevs' contains both "netdev" and "dummy" dpifs. + * If the class doesn't match, skip this dpif. */ + continue; + } sset_add(all_dps, node->name); } ovs_mutex_unlock(&dp_netdev_mutex); @@ -426,7 +400,7 @@ create_dpif_netdev(struct dp_netdev *dp) * Return ODPP_NONE on failure. */ static odp_port_t choose_port(struct dp_netdev *dp, const char *name) - OVS_REQ_RDLOCK(dp->port_rwlock) + OVS_REQUIRES(dp->port_mutex) { uint32_t port_no; @@ -482,20 +456,24 @@ create_dp_netdev(const char *name, const struct dpif_class *class, ovs_mutex_init(&dp->flow_mutex); classifier_init(&dp->cls, NULL); - hmap_init(&dp->flow_table); - - fat_rwlock_init(&dp->queue_rwlock); + cmap_init(&dp->flow_table); ovsthread_stats_init(&dp->stats); - ovs_rwlock_init(&dp->port_rwlock); - hmap_init(&dp->ports); + ovs_mutex_init(&dp->port_mutex); + cmap_init(&dp->ports); dp->port_seq = seq_create(); latch_init(&dp->exit_latch); + fat_rwlock_init(&dp->upcall_rwlock); - ovs_rwlock_wrlock(&dp->port_rwlock); + /* Disable upcalls by default. */ + dp_netdev_disable_upcall(dp); + dp->upcall_aux = NULL; + dp->upcall_cb = NULL; + + ovs_mutex_lock(&dp->port_mutex); error = do_add_port(dp, name, "internal", ODPP_LOCAL); - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp->port_mutex); if (error) { dp_netdev_free(dp); return error; @@ -523,38 +501,20 @@ dpif_netdev_open(const struct dpif_class *class, const char *name, } if (!error) { *dpifp = create_dpif_netdev(dp); + dp->dpif = *dpifp; } ovs_mutex_unlock(&dp_netdev_mutex); return error; } -static void -dp_netdev_purge_queues(struct dp_netdev *dp) - OVS_REQ_WRLOCK(dp->queue_rwlock) -{ - int i; - - for (i = 0; i < dp->n_handlers; i++) { - struct dp_netdev_queue *q = &dp->handler_queues[i]; - - ovs_mutex_lock(&q->mutex); - while (q->tail != q->head) { - struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK]; - ofpbuf_uninit(&u->upcall.packet); - ofpbuf_uninit(&u->buf); - } - ovs_mutex_unlock(&q->mutex); - } -} - /* Requires dp_netdev_mutex so that we can't get a new reference to 'dp' * through the 'dp_netdevs' shash while freeing 'dp'. */ static void dp_netdev_free(struct dp_netdev *dp) OVS_REQUIRES(dp_netdev_mutex) { - struct dp_netdev_port *port, *next; + struct dp_netdev_port *port; struct dp_netdev_stats *bucket; int i; @@ -564,11 +524,11 @@ dp_netdev_free(struct dp_netdev *dp) free(dp->pmd_threads); dp_netdev_flow_flush(dp); - ovs_rwlock_wrlock(&dp->port_rwlock); - HMAP_FOR_EACH_SAFE (port, next, node, &dp->ports) { - do_del_port(dp, port->port_no); + ovs_mutex_lock(&dp->port_mutex); + CMAP_FOR_EACH (port, node, &dp->ports) { + do_del_port(dp, port); } - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp->port_mutex); OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &dp->stats) { ovs_mutex_destroy(&bucket->mutex); @@ -576,17 +536,12 @@ dp_netdev_free(struct dp_netdev *dp) } ovsthread_stats_destroy(&dp->stats); - fat_rwlock_wrlock(&dp->queue_rwlock); - dp_netdev_destroy_all_queues(dp); - fat_rwlock_unlock(&dp->queue_rwlock); - - fat_rwlock_destroy(&dp->queue_rwlock); - classifier_destroy(&dp->cls); - hmap_destroy(&dp->flow_table); + cmap_destroy(&dp->flow_table); ovs_mutex_destroy(&dp->flow_mutex); seq_destroy(dp->port_seq); - hmap_destroy(&dp->ports); + cmap_destroy(&dp->ports); + fat_rwlock_destroy(&dp->upcall_rwlock); latch_destroy(&dp->exit_latch); free(CONST_CAST(char *, dp->name)); free(dp); @@ -599,7 +554,7 @@ dp_netdev_unref(struct dp_netdev *dp) /* Take dp_netdev_mutex so that, if dp->ref_cnt falls to zero, we can't * get a new reference to 'dp' through the 'dp_netdevs' shash. */ ovs_mutex_lock(&dp_netdev_mutex); - if (ovs_refcount_unref(&dp->ref_cnt) == 1) { + if (ovs_refcount_unref_relaxed(&dp->ref_cnt) == 1) { dp_netdev_free(dp); } ovs_mutex_unlock(&dp_netdev_mutex); @@ -621,7 +576,7 @@ dpif_netdev_destroy(struct dpif *dpif) struct dp_netdev *dp = get_dp_netdev(dpif); if (!atomic_flag_test_and_set(&dp->destroyed)) { - if (ovs_refcount_unref(&dp->ref_cnt) == 1) { + if (ovs_refcount_unref_relaxed(&dp->ref_cnt) == 1) { /* Can't happen: 'dpif' still owns a reference to 'dp'. */ OVS_NOT_REACHED(); } @@ -637,9 +592,7 @@ dpif_netdev_get_stats(const struct dpif *dpif, struct dpif_dp_stats *stats) struct dp_netdev_stats *bucket; size_t i; - fat_rwlock_rdlock(&dp->cls.rwlock); - stats->n_flows = hmap_count(&dp->flow_table); - fat_rwlock_unlock(&dp->cls.rwlock); + stats->n_flows = cmap_count(&dp->flow_table); stats->n_hit = stats->n_missed = stats->n_lost = 0; OVSTHREAD_STATS_FOR_EACH_BUCKET (bucket, i, &dp->stats) { @@ -662,16 +615,22 @@ dp_netdev_reload_pmd_threads(struct dp_netdev *dp) for (i = 0; i < dp->n_pmd_threads; i++) { struct pmd_thread *f = &dp->pmd_threads[i]; - int id; + int old_seq; + + atomic_add_relaxed(&f->change_seq, 1, &old_seq); + } +} - atomic_add(&f->change_seq, 1, &id); - } +static uint32_t +hash_port_no(odp_port_t port_no) +{ + return hash_int(odp_to_u32(port_no), 0); } static int do_add_port(struct dp_netdev *dp, const char *devname, const char *type, odp_port_t port_no) - OVS_REQ_WRLOCK(dp->port_rwlock) + OVS_REQUIRES(dp->port_mutex) { struct netdev_saved_flags *sf; struct dp_netdev_port *port; @@ -710,6 +669,9 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type, VLOG_ERR("%s: cannot receive packets on this network device (%s)", devname, ovs_strerror(errno)); netdev_close(netdev); + free(port->type); + free(port->rxq); + free(port); return error; } } @@ -720,6 +682,7 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type, netdev_rxq_close(port->rxq[i]); } netdev_close(netdev); + free(port->type); free(port->rxq); free(port); return error; @@ -728,12 +691,12 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type, if (netdev_is_pmd(netdev)) { dp->pmd_count++; - dp_netdev_set_pmd_threads(dp, NR_THREADS); + dp_netdev_set_pmd_threads(dp, NR_PMD_THREADS); dp_netdev_reload_pmd_threads(dp); } ovs_refcount_init(&port->ref_cnt); - hmap_insert(&dp->ports, &port->node, hash_int(odp_to_u32(port_no), 0)); + cmap_insert(&dp->ports, &port->node, hash_port_no(port_no)); seq_change(dp->port_seq); return 0; @@ -749,7 +712,7 @@ dpif_netdev_port_add(struct dpif *dpif, struct netdev *netdev, odp_port_t port_no; int error; - ovs_rwlock_wrlock(&dp->port_rwlock); + ovs_mutex_lock(&dp->port_mutex); dpif_port = netdev_vport_get_dpif_port(netdev, namebuf, sizeof namebuf); if (*port_nop != ODPP_NONE) { port_no = *port_nop; @@ -762,7 +725,7 @@ dpif_netdev_port_add(struct dpif *dpif, struct netdev *netdev, *port_nop = port_no; error = do_add_port(dp, dpif_port, netdev_get_type(netdev), port_no); } - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp->port_mutex); return error; } @@ -773,9 +736,18 @@ dpif_netdev_port_del(struct dpif *dpif, odp_port_t port_no) struct dp_netdev *dp = get_dp_netdev(dpif); int error; - ovs_rwlock_wrlock(&dp->port_rwlock); - error = port_no == ODPP_LOCAL ? EINVAL : do_del_port(dp, port_no); - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_lock(&dp->port_mutex); + if (port_no == ODPP_LOCAL) { + error = EINVAL; + } else { + struct dp_netdev_port *port; + + error = get_port_by_number(dp, port_no, &port); + if (!error) { + do_del_port(dp, port); + } + } + ovs_mutex_unlock(&dp->port_mutex); return error; } @@ -788,12 +760,10 @@ is_valid_port_number(odp_port_t port_no) static struct dp_netdev_port * dp_netdev_lookup_port(const struct dp_netdev *dp, odp_port_t port_no) - OVS_REQ_RDLOCK(dp->port_rwlock) { struct dp_netdev_port *port; - HMAP_FOR_EACH_IN_BUCKET (port, node, hash_int(odp_to_u32(port_no), 0), - &dp->ports) { + CMAP_FOR_EACH_WITH_HASH (port, node, hash_port_no(port_no), &dp->ports) { if (port->port_no == port_no) { return port; } @@ -804,7 +774,6 @@ dp_netdev_lookup_port(const struct dp_netdev *dp, odp_port_t port_no) static int get_port_by_number(struct dp_netdev *dp, odp_port_t port_no, struct dp_netdev_port **portp) - OVS_REQ_RDLOCK(dp->port_rwlock) { if (!is_valid_port_number(port_no)) { *portp = NULL; @@ -824,30 +793,38 @@ port_ref(struct dp_netdev_port *port) } static void -port_unref(struct dp_netdev_port *port) +port_destroy__(struct dp_netdev_port *port) { - if (port && ovs_refcount_unref(&port->ref_cnt) == 1) { - int i; + int n_rxq = netdev_n_rxq(port->netdev); + int i; - netdev_close(port->netdev); - netdev_restore_flags(port->sf); + netdev_close(port->netdev); + netdev_restore_flags(port->sf); - for (i = 0; i < netdev_n_rxq(port->netdev); i++) { - netdev_rxq_close(port->rxq[i]); - } - free(port->type); - free(port); + for (i = 0; i < n_rxq; i++) { + netdev_rxq_close(port->rxq[i]); + } + free(port->rxq); + free(port->type); + free(port); +} + +static void +port_unref(struct dp_netdev_port *port) +{ + if (port && ovs_refcount_unref_relaxed(&port->ref_cnt) == 1) { + ovsrcu_postpone(port_destroy__, port); } } static int get_port_by_name(struct dp_netdev *dp, const char *devname, struct dp_netdev_port **portp) - OVS_REQ_RDLOCK(dp->port_rwlock) + OVS_REQUIRES(dp->port_mutex) { struct dp_netdev_port *port; - HMAP_FOR_EACH (port, node, &dp->ports) { + CMAP_FOR_EACH (port, node, &dp->ports) { if (!strcmp(netdev_get_name(port->netdev), devname)) { *portp = port; return 0; @@ -856,26 +833,17 @@ get_port_by_name(struct dp_netdev *dp, return ENOENT; } -static int -do_del_port(struct dp_netdev *dp, odp_port_t port_no) - OVS_REQ_WRLOCK(dp->port_rwlock) +static void +do_del_port(struct dp_netdev *dp, struct dp_netdev_port *port) + OVS_REQUIRES(dp->port_mutex) { - struct dp_netdev_port *port; - int error; - - error = get_port_by_number(dp, port_no, &port); - if (error) { - return error; - } - - hmap_remove(&dp->ports, &port->node); + cmap_remove(&dp->ports, &port->node, hash_odp_port(port->port_no)); seq_change(dp->port_seq); if (netdev_is_pmd(port->netdev)) { dp_netdev_reload_pmd_threads(dp); } port_unref(port); - return 0; } static void @@ -895,12 +863,10 @@ dpif_netdev_port_query_by_number(const struct dpif *dpif, odp_port_t port_no, struct dp_netdev_port *port; int error; - ovs_rwlock_rdlock(&dp->port_rwlock); error = get_port_by_number(dp, port_no, &port); if (!error && dpif_port) { answer_port_query(port, dpif_port); } - ovs_rwlock_unlock(&dp->port_rwlock); return error; } @@ -913,12 +879,12 @@ dpif_netdev_port_query_by_name(const struct dpif *dpif, const char *devname, struct dp_netdev_port *port; int error; - ovs_rwlock_rdlock(&dp->port_rwlock); + ovs_mutex_lock(&dp->port_mutex); error = get_port_by_name(dp, devname, &port); if (!error && dpif_port) { answer_port_query(port, dpif_port); } - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp->port_mutex); return error; } @@ -937,34 +903,38 @@ dp_netdev_flow_free(struct dp_netdev_flow *flow) cls_rule_destroy(CONST_CAST(struct cls_rule *, &flow->cr)); dp_netdev_actions_free(dp_netdev_flow_get_actions(flow)); - ovs_mutex_destroy(&flow->mutex); free(flow); } +static void dp_netdev_flow_unref(struct dp_netdev_flow *flow) +{ + if (ovs_refcount_unref_relaxed(&flow->ref_cnt) == 1) { + ovsrcu_postpone(dp_netdev_flow_free, flow); + } +} + static void dp_netdev_remove_flow(struct dp_netdev *dp, struct dp_netdev_flow *flow) - OVS_REQ_WRLOCK(dp->cls.rwlock) OVS_REQUIRES(dp->flow_mutex) { struct cls_rule *cr = CONST_CAST(struct cls_rule *, &flow->cr); - struct hmap_node *node = CONST_CAST(struct hmap_node *, &flow->node); + struct cmap_node *node = CONST_CAST(struct cmap_node *, &flow->node); classifier_remove(&dp->cls, cr); - hmap_remove(&dp->flow_table, node); - ovsrcu_postpone(dp_netdev_flow_free, flow); + cmap_remove(&dp->flow_table, node, flow_hash(&flow->flow, 0)); + + dp_netdev_flow_unref(flow); } static void dp_netdev_flow_flush(struct dp_netdev *dp) { - struct dp_netdev_flow *netdev_flow, *next; + struct dp_netdev_flow *netdev_flow; ovs_mutex_lock(&dp->flow_mutex); - fat_rwlock_wrlock(&dp->cls.rwlock); - HMAP_FOR_EACH_SAFE (netdev_flow, next, node, &dp->flow_table) { + CMAP_FOR_EACH (netdev_flow, node, &dp->flow_table) { dp_netdev_remove_flow(dp, netdev_flow); } - fat_rwlock_unlock(&dp->cls.rwlock); ovs_mutex_unlock(&dp->flow_mutex); } @@ -978,8 +948,7 @@ dpif_netdev_flow_flush(struct dpif *dpif) } struct dp_netdev_port_state { - uint32_t bucket; - uint32_t offset; + struct cmap_position position; char *name; }; @@ -996,11 +965,10 @@ dpif_netdev_port_dump_next(const struct dpif *dpif, void *state_, { struct dp_netdev_port_state *state = state_; struct dp_netdev *dp = get_dp_netdev(dpif); - struct hmap_node *node; + struct cmap_node *node; int retval; - ovs_rwlock_rdlock(&dp->port_rwlock); - node = hmap_at_position(&dp->ports, &state->bucket, &state->offset); + node = cmap_next_position(&dp->ports, &state->position); if (node) { struct dp_netdev_port *port; @@ -1016,7 +984,6 @@ dpif_netdev_port_dump_next(const struct dpif *dpif, void *state_, } else { retval = EOF; } - ovs_rwlock_unlock(&dp->port_rwlock); return retval; } @@ -1063,25 +1030,23 @@ dp_netdev_flow_cast(const struct cls_rule *cr) } static struct dp_netdev_flow * -dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct flow *flow) - OVS_EXCLUDED(dp->cls.rwlock) +dp_netdev_lookup_flow(const struct dp_netdev *dp, const struct miniflow *key) { struct dp_netdev_flow *netdev_flow; + struct cls_rule *rule; - fat_rwlock_rdlock(&dp->cls.rwlock); - netdev_flow = dp_netdev_flow_cast(classifier_lookup(&dp->cls, flow, NULL)); - fat_rwlock_unlock(&dp->cls.rwlock); + classifier_lookup_miniflow_batch(&dp->cls, &key, &rule, 1); + netdev_flow = dp_netdev_flow_cast(rule); return netdev_flow; } static struct dp_netdev_flow * dp_netdev_find_flow(const struct dp_netdev *dp, const struct flow *flow) - OVS_REQ_RDLOCK(dp->cls.rwlock) { struct dp_netdev_flow *netdev_flow; - HMAP_FOR_EACH_WITH_HASH (netdev_flow, node, flow_hash(flow, 0), + CMAP_FOR_EACH_WITH_HASH (netdev_flow, node, flow_hash(flow, 0), &dp->flow_table) { if (flow_equal(&netdev_flow->flow, flow)) { return netdev_flow; @@ -1092,7 +1057,7 @@ dp_netdev_find_flow(const struct dp_netdev *dp, const struct flow *flow) } static void -get_dpif_flow_stats(struct dp_netdev_flow *netdev_flow, +get_dpif_flow_stats(const struct dp_netdev_flow *netdev_flow, struct dpif_flow_stats *stats) { struct dp_netdev_flow_stats *bucket; @@ -1109,6 +1074,27 @@ get_dpif_flow_stats(struct dp_netdev_flow *netdev_flow, } } +static void +dp_netdev_flow_to_dpif_flow(const struct dp_netdev_flow *netdev_flow, + struct ofpbuf *buffer, struct dpif_flow *flow) +{ + struct flow_wildcards wc; + struct dp_netdev_actions *actions; + + minimask_expand(&netdev_flow->cr.match.mask, &wc); + odp_flow_key_from_mask(buffer, &wc.masks, &netdev_flow->flow, + odp_to_u32(wc.masks.in_port.odp_port), + SIZE_MAX, true); + flow->mask = ofpbuf_data(buffer); + flow->mask_len = ofpbuf_size(buffer); + + actions = dp_netdev_flow_get_actions(netdev_flow); + flow->actions = actions->actions; + flow->actions_len = actions->size; + + get_dpif_flow_stats(netdev_flow, &flow->stats); +} + static int dpif_netdev_mask_from_nlattrs(const struct nlattr *key, uint32_t key_len, const struct nlattr *mask_key, @@ -1202,35 +1188,22 @@ dpif_netdev_flow_from_nlattrs(const struct nlattr *key, uint32_t key_len, } static int -dpif_netdev_flow_get(const struct dpif *dpif, - const struct nlattr *nl_key, size_t nl_key_len, - struct ofpbuf **actionsp, struct dpif_flow_stats *stats) +dpif_netdev_flow_get(const struct dpif *dpif, const struct dpif_flow_get *get) { struct dp_netdev *dp = get_dp_netdev(dpif); struct dp_netdev_flow *netdev_flow; struct flow key; int error; - error = dpif_netdev_flow_from_nlattrs(nl_key, nl_key_len, &key); + error = dpif_netdev_flow_from_nlattrs(get->key, get->key_len, &key); if (error) { return error; } - fat_rwlock_rdlock(&dp->cls.rwlock); netdev_flow = dp_netdev_find_flow(dp, &key); - fat_rwlock_unlock(&dp->cls.rwlock); if (netdev_flow) { - if (stats) { - get_dpif_flow_stats(netdev_flow, stats); - } - - if (actionsp) { - struct dp_netdev_actions *actions; - - actions = dp_netdev_flow_get_actions(netdev_flow); - *actionsp = ofpbuf_clone_data(actions->actions, actions->size); - } + dp_netdev_flow_to_dpif_flow(netdev_flow, get->buffer, get->flow); } else { error = ENOENT; } @@ -1239,35 +1212,42 @@ dpif_netdev_flow_get(const struct dpif *dpif, } static int -dp_netdev_flow_add(struct dp_netdev *dp, const struct flow *flow, - const struct flow_wildcards *wc, - const struct nlattr *actions, - size_t actions_len) +dp_netdev_flow_add(struct dp_netdev *dp, struct match *match, + const struct nlattr *actions, size_t actions_len) OVS_REQUIRES(dp->flow_mutex) { struct dp_netdev_flow *netdev_flow; - struct match match; netdev_flow = xzalloc(sizeof *netdev_flow); - *CONST_CAST(struct flow *, &netdev_flow->flow) = *flow; + *CONST_CAST(struct flow *, &netdev_flow->flow) = match->flow; - ovs_mutex_init(&netdev_flow->mutex); + ovs_refcount_init(&netdev_flow->ref_cnt); ovsthread_stats_init(&netdev_flow->stats); ovsrcu_set(&netdev_flow->actions, dp_netdev_actions_create(actions, actions_len)); - match_init(&match, flow, wc); cls_rule_init(CONST_CAST(struct cls_rule *, &netdev_flow->cr), - &match, NETDEV_RULE_PRIORITY); - fat_rwlock_wrlock(&dp->cls.rwlock); + match, NETDEV_RULE_PRIORITY); + cmap_insert(&dp->flow_table, + CONST_CAST(struct cmap_node *, &netdev_flow->node), + flow_hash(&match->flow, 0)); classifier_insert(&dp->cls, CONST_CAST(struct cls_rule *, &netdev_flow->cr)); - hmap_insert(&dp->flow_table, - CONST_CAST(struct hmap_node *, &netdev_flow->node), - flow_hash(flow, 0)); - fat_rwlock_unlock(&dp->cls.rwlock); + + if (OVS_UNLIKELY(VLOG_IS_DBG_ENABLED())) { + struct ds ds = DS_EMPTY_INITIALIZER; + + ds_put_cstr(&ds, "flow_add: "); + match_format(match, &ds, OFP_DEFAULT_PRIORITY); + ds_put_cstr(&ds, ", actions:"); + format_odp_actions(&ds, actions, actions_len); + + VLOG_DBG_RL(&upcall_rl, "%s", ds_cstr(&ds)); + + ds_destroy(&ds); + } return 0; } @@ -1293,30 +1273,31 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) { struct dp_netdev *dp = get_dp_netdev(dpif); struct dp_netdev_flow *netdev_flow; - struct flow flow; - struct flow_wildcards wc; + struct miniflow miniflow; + struct match match; int error; - error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &flow); + error = dpif_netdev_flow_from_nlattrs(put->key, put->key_len, &match.flow); if (error) { return error; } error = dpif_netdev_mask_from_nlattrs(put->key, put->key_len, put->mask, put->mask_len, - &flow, &wc.masks); + &match.flow, &match.wc.masks); if (error) { return error; } + miniflow_init(&miniflow, &match.flow); ovs_mutex_lock(&dp->flow_mutex); - netdev_flow = dp_netdev_lookup_flow(dp, &flow); + netdev_flow = dp_netdev_lookup_flow(dp, &miniflow); if (!netdev_flow) { if (put->flags & DPIF_FP_CREATE) { - if (hmap_count(&dp->flow_table) < MAX_FLOWS) { + if (cmap_count(&dp->flow_table) < MAX_FLOWS) { if (put->stats) { memset(put->stats, 0, sizeof *put->stats); } - error = dp_netdev_flow_add(dp, &flow, &wc, put->actions, + error = dp_netdev_flow_add(dp, &match, put->actions, put->actions_len); } else { error = EFBIG; @@ -1326,7 +1307,7 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) } } else { if (put->flags & DPIF_FP_MODIFY - && flow_equal(&flow, &netdev_flow->flow)) { + && flow_equal(&match.flow, &netdev_flow->flow)) { struct dp_netdev_actions *new_actions; struct dp_netdev_actions *old_actions; @@ -1352,6 +1333,7 @@ dpif_netdev_flow_put(struct dpif *dpif, const struct dpif_flow_put *put) } } ovs_mutex_unlock(&dp->flow_mutex); + miniflow_destroy(&miniflow); return error; } @@ -1370,7 +1352,6 @@ dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del) } ovs_mutex_lock(&dp->flow_mutex); - fat_rwlock_wrlock(&dp->cls.rwlock); netdev_flow = dp_netdev_find_flow(dp, &key); if (netdev_flow) { if (del->stats) { @@ -1380,240 +1361,204 @@ dpif_netdev_flow_del(struct dpif *dpif, const struct dpif_flow_del *del) } else { error = ENOENT; } - fat_rwlock_unlock(&dp->cls.rwlock); ovs_mutex_unlock(&dp->flow_mutex); return error; } -struct dp_netdev_flow_state { - struct dp_netdev_actions *actions; - struct odputil_keybuf keybuf; - struct odputil_keybuf maskbuf; - struct dpif_flow_stats stats; -}; - -struct dp_netdev_flow_iter { - uint32_t bucket; - uint32_t offset; +struct dpif_netdev_flow_dump { + struct dpif_flow_dump up; + struct cmap_position pos; int status; struct ovs_mutex mutex; }; -static void -dpif_netdev_flow_dump_state_init(void **statep) +static struct dpif_netdev_flow_dump * +dpif_netdev_flow_dump_cast(struct dpif_flow_dump *dump) { - struct dp_netdev_flow_state *state; - - *statep = state = xmalloc(sizeof *state); - state->actions = NULL; + return CONTAINER_OF(dump, struct dpif_netdev_flow_dump, up); } -static void -dpif_netdev_flow_dump_state_uninit(void *state_) +static struct dpif_flow_dump * +dpif_netdev_flow_dump_create(const struct dpif *dpif_) { - struct dp_netdev_flow_state *state = state_; + struct dpif_netdev_flow_dump *dump; - free(state); + dump = xmalloc(sizeof *dump); + dpif_flow_dump_init(&dump->up, dpif_); + memset(&dump->pos, 0, sizeof dump->pos); + dump->status = 0; + ovs_mutex_init(&dump->mutex); + + return &dump->up; } static int -dpif_netdev_flow_dump_start(const struct dpif *dpif OVS_UNUSED, void **iterp) +dpif_netdev_flow_dump_destroy(struct dpif_flow_dump *dump_) { - struct dp_netdev_flow_iter *iter; + struct dpif_netdev_flow_dump *dump = dpif_netdev_flow_dump_cast(dump_); - *iterp = iter = xmalloc(sizeof *iter); - iter->bucket = 0; - iter->offset = 0; - iter->status = 0; - ovs_mutex_init(&iter->mutex); + ovs_mutex_destroy(&dump->mutex); + free(dump); return 0; } -/* XXX the caller must use 'actions' without quiescing */ -static int -dpif_netdev_flow_dump_next(const struct dpif *dpif, void *iter_, void *state_, - const struct nlattr **key, size_t *key_len, - const struct nlattr **mask, size_t *mask_len, - const struct nlattr **actions, size_t *actions_len, - const struct dpif_flow_stats **stats) -{ - struct dp_netdev_flow_iter *iter = iter_; - struct dp_netdev_flow_state *state = state_; - struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_flow *netdev_flow; - int error; +struct dpif_netdev_flow_dump_thread { + struct dpif_flow_dump_thread up; + struct dpif_netdev_flow_dump *dump; + struct odputil_keybuf keybuf[FLOW_DUMP_MAX_BATCH]; + struct odputil_keybuf maskbuf[FLOW_DUMP_MAX_BATCH]; +}; - ovs_mutex_lock(&iter->mutex); - error = iter->status; - if (!error) { - struct hmap_node *node; +static struct dpif_netdev_flow_dump_thread * +dpif_netdev_flow_dump_thread_cast(struct dpif_flow_dump_thread *thread) +{ + return CONTAINER_OF(thread, struct dpif_netdev_flow_dump_thread, up); +} - fat_rwlock_rdlock(&dp->cls.rwlock); - node = hmap_at_position(&dp->flow_table, &iter->bucket, &iter->offset); - if (node) { - netdev_flow = CONTAINER_OF(node, struct dp_netdev_flow, node); - } - fat_rwlock_unlock(&dp->cls.rwlock); - if (!node) { - iter->status = error = EOF; - } - } - ovs_mutex_unlock(&iter->mutex); - if (error) { - return error; - } +static struct dpif_flow_dump_thread * +dpif_netdev_flow_dump_thread_create(struct dpif_flow_dump *dump_) +{ + struct dpif_netdev_flow_dump *dump = dpif_netdev_flow_dump_cast(dump_); + struct dpif_netdev_flow_dump_thread *thread; - if (key) { - struct ofpbuf buf; + thread = xmalloc(sizeof *thread); + dpif_flow_dump_thread_init(&thread->up, &dump->up); + thread->dump = dump; + return &thread->up; +} - ofpbuf_use_stack(&buf, &state->keybuf, sizeof state->keybuf); - odp_flow_key_from_flow(&buf, &netdev_flow->flow, - netdev_flow->flow.in_port.odp_port); +static void +dpif_netdev_flow_dump_thread_destroy(struct dpif_flow_dump_thread *thread_) +{ + struct dpif_netdev_flow_dump_thread *thread + = dpif_netdev_flow_dump_thread_cast(thread_); - *key = ofpbuf_data(&buf); - *key_len = ofpbuf_size(&buf); + free(thread); +} + +static int +dpif_netdev_flow_dump_next(struct dpif_flow_dump_thread *thread_, + struct dpif_flow *flows, int max_flows) +{ + struct dpif_netdev_flow_dump_thread *thread + = dpif_netdev_flow_dump_thread_cast(thread_); + struct dpif_netdev_flow_dump *dump = thread->dump; + struct dpif_netdev *dpif = dpif_netdev_cast(thread->up.dpif); + struct dp_netdev_flow *netdev_flows[FLOW_DUMP_MAX_BATCH]; + struct dp_netdev *dp = get_dp_netdev(&dpif->dpif); + int n_flows = 0; + int i; + + ovs_mutex_lock(&dump->mutex); + if (!dump->status) { + for (n_flows = 0; n_flows < MIN(max_flows, FLOW_DUMP_MAX_BATCH); + n_flows++) { + struct cmap_node *node; + + node = cmap_next_position(&dp->flow_table, &dump->pos); + if (!node) { + dump->status = EOF; + break; + } + netdev_flows[n_flows] = CONTAINER_OF(node, struct dp_netdev_flow, + node); + } } + ovs_mutex_unlock(&dump->mutex); - if (key && mask) { - struct ofpbuf buf; + for (i = 0; i < n_flows; i++) { + struct odputil_keybuf *maskbuf = &thread->maskbuf[i]; + struct odputil_keybuf *keybuf = &thread->keybuf[i]; + struct dp_netdev_flow *netdev_flow = netdev_flows[i]; + struct dpif_flow *f = &flows[i]; + struct dp_netdev_actions *dp_actions; struct flow_wildcards wc; + struct ofpbuf buf; - ofpbuf_use_stack(&buf, &state->maskbuf, sizeof state->maskbuf); minimask_expand(&netdev_flow->cr.match.mask, &wc); - odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow, - odp_to_u32(wc.masks.in_port.odp_port), - SIZE_MAX); - *mask = ofpbuf_data(&buf); - *mask_len = ofpbuf_size(&buf); - } + /* Key. */ + ofpbuf_use_stack(&buf, keybuf, sizeof *keybuf); + odp_flow_key_from_flow(&buf, &netdev_flow->flow, &wc.masks, + netdev_flow->flow.in_port.odp_port, true); + f->key = ofpbuf_data(&buf); + f->key_len = ofpbuf_size(&buf); - if (actions || stats) { - state->actions = NULL; + /* Mask. */ + ofpbuf_use_stack(&buf, maskbuf, sizeof *maskbuf); + odp_flow_key_from_mask(&buf, &wc.masks, &netdev_flow->flow, + odp_to_u32(wc.masks.in_port.odp_port), + SIZE_MAX, true); + f->mask = ofpbuf_data(&buf); + f->mask_len = ofpbuf_size(&buf); - if (actions) { - state->actions = dp_netdev_flow_get_actions(netdev_flow); - *actions = state->actions->actions; - *actions_len = state->actions->size; - } + /* Actions. */ + dp_actions = dp_netdev_flow_get_actions(netdev_flow); + f->actions = dp_actions->actions; + f->actions_len = dp_actions->size; - if (stats) { - get_dpif_flow_stats(netdev_flow, &state->stats); - *stats = &state->stats; - } + /* Stats. */ + get_dpif_flow_stats(netdev_flow, &f->stats); } - return 0; -} - -static int -dpif_netdev_flow_dump_done(const struct dpif *dpif OVS_UNUSED, void *iter_) -{ - struct dp_netdev_flow_iter *iter = iter_; - - ovs_mutex_destroy(&iter->mutex); - free(iter); - return 0; + return n_flows; } static int dpif_netdev_execute(struct dpif *dpif, struct dpif_execute *execute) { struct dp_netdev *dp = get_dp_netdev(dpif); + struct dpif_packet packet, *pp; struct pkt_metadata *md = &execute->md; - struct flow key; if (ofpbuf_size(execute->packet) < ETH_HEADER_LEN || ofpbuf_size(execute->packet) > UINT16_MAX) { return EINVAL; } - /* Extract flow key. */ - flow_extract(execute->packet, md, &key); + packet.ofpbuf = *execute->packet; + pp = &packet; - ovs_rwlock_rdlock(&dp->port_rwlock); - dp_netdev_execute_actions(dp, &key, execute->packet, false, md, + dp_netdev_execute_actions(dp, &pp, 1, false, md, execute->actions, execute->actions_len); - ovs_rwlock_unlock(&dp->port_rwlock); + + /* Even though may_steal is set to false, some actions could modify or + * reallocate the ofpbuf memory. We need to pass those changes to the + * caller */ + *execute->packet = packet.ofpbuf; return 0; } static void -dp_netdev_destroy_all_queues(struct dp_netdev *dp) - OVS_REQ_WRLOCK(dp->queue_rwlock) +dpif_netdev_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops) { size_t i; - dp_netdev_purge_queues(dp); - - for (i = 0; i < dp->n_handlers; i++) { - struct dp_netdev_queue *q = &dp->handler_queues[i]; + for (i = 0; i < n_ops; i++) { + struct dpif_op *op = ops[i]; - ovs_mutex_destroy(&q->mutex); - seq_destroy(q->seq); - } - free(dp->handler_queues); - dp->handler_queues = NULL; - dp->n_handlers = 0; -} - -static void -dp_netdev_refresh_queues(struct dp_netdev *dp, uint32_t n_handlers) - OVS_REQ_WRLOCK(dp->queue_rwlock) -{ - if (dp->n_handlers != n_handlers) { - size_t i; - - dp_netdev_destroy_all_queues(dp); + switch (op->type) { + case DPIF_OP_FLOW_PUT: + op->error = dpif_netdev_flow_put(dpif, &op->u.flow_put); + break; - dp->n_handlers = n_handlers; - dp->handler_queues = xzalloc(n_handlers * sizeof *dp->handler_queues); + case DPIF_OP_FLOW_DEL: + op->error = dpif_netdev_flow_del(dpif, &op->u.flow_del); + break; - for (i = 0; i < n_handlers; i++) { - struct dp_netdev_queue *q = &dp->handler_queues[i]; + case DPIF_OP_EXECUTE: + op->error = dpif_netdev_execute(dpif, &op->u.execute); + break; - ovs_mutex_init(&q->mutex); - q->seq = seq_create(); + case DPIF_OP_FLOW_GET: + op->error = dpif_netdev_flow_get(dpif, &op->u.flow_get); + break; } } } -static int -dpif_netdev_recv_set(struct dpif *dpif, bool enable) -{ - struct dp_netdev *dp = get_dp_netdev(dpif); - - if ((dp->handler_queues != NULL) == enable) { - return 0; - } - - fat_rwlock_wrlock(&dp->queue_rwlock); - if (!enable) { - dp_netdev_destroy_all_queues(dp); - } else { - dp_netdev_refresh_queues(dp, 1); - } - fat_rwlock_unlock(&dp->queue_rwlock); - - return 0; -} - -static int -dpif_netdev_handlers_set(struct dpif *dpif, uint32_t n_handlers) -{ - struct dp_netdev *dp = get_dp_netdev(dpif); - - fat_rwlock_wrlock(&dp->queue_rwlock); - if (dp->handler_queues) { - dp_netdev_refresh_queues(dp, n_handlers); - } - fat_rwlock_unlock(&dp->queue_rwlock); - - return 0; -} - static int dpif_netdev_queue_to_priority(const struct dpif *dpif OVS_UNUSED, uint32_t queue_id, uint32_t *priority) @@ -1622,97 +1567,6 @@ dpif_netdev_queue_to_priority(const struct dpif *dpif OVS_UNUSED, return 0; } -static bool -dp_netdev_recv_check(const struct dp_netdev *dp, const uint32_t handler_id) - OVS_REQ_RDLOCK(dp->queue_rwlock) -{ - static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); - - if (!dp->handler_queues) { - VLOG_WARN_RL(&rl, "receiving upcall disabled"); - return false; - } - - if (handler_id >= dp->n_handlers) { - VLOG_WARN_RL(&rl, "handler index out of bound"); - return false; - } - - return true; -} - -static int -dpif_netdev_recv(struct dpif *dpif, uint32_t handler_id, - struct dpif_upcall *upcall, struct ofpbuf *buf) -{ - struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_queue *q; - int error = 0; - - fat_rwlock_rdlock(&dp->queue_rwlock); - - if (!dp_netdev_recv_check(dp, handler_id)) { - error = EAGAIN; - goto out; - } - - q = &dp->handler_queues[handler_id]; - ovs_mutex_lock(&q->mutex); - if (q->head != q->tail) { - struct dp_netdev_upcall *u = &q->upcalls[q->tail++ & QUEUE_MASK]; - - *upcall = u->upcall; - - ofpbuf_uninit(buf); - *buf = u->buf; - } else { - error = EAGAIN; - } - ovs_mutex_unlock(&q->mutex); - -out: - fat_rwlock_unlock(&dp->queue_rwlock); - - return error; -} - -static void -dpif_netdev_recv_wait(struct dpif *dpif, uint32_t handler_id) -{ - struct dp_netdev *dp = get_dp_netdev(dpif); - struct dp_netdev_queue *q; - uint64_t seq; - - fat_rwlock_rdlock(&dp->queue_rwlock); - - if (!dp_netdev_recv_check(dp, handler_id)) { - goto out; - } - - q = &dp->handler_queues[handler_id]; - ovs_mutex_lock(&q->mutex); - seq = seq_read(q->seq); - if (q->head != q->tail) { - poll_immediate_wake(); - } else { - seq_wait(q->seq, seq); - } - - ovs_mutex_unlock(&q->mutex); - -out: - fat_rwlock_unlock(&dp->queue_rwlock); -} - -static void -dpif_netdev_recv_purge(struct dpif *dpif) -{ - struct dpif_netdev *dpif_netdev = dpif_netdev_cast(dpif); - - fat_rwlock_wrlock(&dpif_netdev->dp->queue_rwlock); - dp_netdev_purge_queues(dpif_netdev->dp); - fat_rwlock_unlock(&dpif_netdev->dp->queue_rwlock); -} /* Creates and returns a new 'struct dp_netdev_actions', with a reference count * of 1, whose actions are a copy of from the 'ofpacts_len' bytes of @@ -1748,17 +1602,12 @@ dp_netdev_process_rxq_port(struct dp_netdev *dp, struct dp_netdev_port *port, struct netdev_rxq *rxq) { - struct ofpbuf *packet[NETDEV_MAX_RX_BATCH]; - int error, c; + struct dpif_packet *packets[NETDEV_MAX_RX_BATCH]; + int error, cnt; - error = netdev_rxq_recv(rxq, packet, &c); + error = netdev_rxq_recv(rxq, packets, &cnt); if (!error) { - struct pkt_metadata md = PKT_METADATA_INITIALIZER(port->port_no); - int i; - - for (i = 0; i < c; i++) { - dp_netdev_port_input(dp, packet[i], &md); - } + dp_netdev_port_input(dp, packets, cnt, port->port_no); } else if (error != EAGAIN && error != EOPNOTSUPP) { static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); @@ -1775,9 +1624,7 @@ dpif_netdev_run(struct dpif *dpif) struct dp_netdev_port *port; struct dp_netdev *dp = get_dp_netdev(dpif); - ovs_rwlock_rdlock(&dp->port_rwlock); - - HMAP_FOR_EACH (port, node, &dp->ports) { + CMAP_FOR_EACH (port, node, &dp->ports) { if (!netdev_is_pmd(port->netdev)) { int i; @@ -1786,8 +1633,6 @@ dpif_netdev_run(struct dpif *dpif) } } } - - ovs_rwlock_unlock(&dp->port_rwlock); } static void @@ -1796,9 +1641,8 @@ dpif_netdev_wait(struct dpif *dpif) struct dp_netdev_port *port; struct dp_netdev *dp = get_dp_netdev(dpif); - ovs_rwlock_rdlock(&dp->port_rwlock); - - HMAP_FOR_EACH (port, node, &dp->ports) { + ovs_mutex_lock(&dp_netdev_mutex); + CMAP_FOR_EACH (port, node, &dp->ports) { if (!netdev_is_pmd(port->netdev)) { int i; @@ -1807,7 +1651,7 @@ dpif_netdev_wait(struct dpif *dpif) } } } - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp_netdev_mutex); } struct rxq_poll { @@ -1827,7 +1671,6 @@ pmd_load_queues(struct pmd_thread *f, int i; /* Simple scheduler for netdev rx polling. */ - ovs_rwlock_rdlock(&dp->port_rwlock); for (i = 0; i < poll_cnt; i++) { port_unref(poll_list[i].port); } @@ -1835,7 +1678,7 @@ pmd_load_queues(struct pmd_thread *f, poll_cnt = 0; index = 0; - HMAP_FOR_EACH (port, node, &f->dp->ports) { + CMAP_FOR_EACH (port, node, &f->dp->ports) { if (netdev_is_pmd(port->netdev)) { int i; @@ -1853,7 +1696,6 @@ pmd_load_queues(struct pmd_thread *f, } } - ovs_rwlock_unlock(&dp->port_rwlock); *ppoll_list = poll_list; return poll_cnt; } @@ -1865,22 +1707,18 @@ pmd_thread_main(void *f_) struct dp_netdev *dp = f->dp; unsigned int lc = 0; struct rxq_poll *poll_list; - unsigned int port_seq; + unsigned int port_seq = PMD_INITIAL_SEQ; int poll_cnt; int i; - f->name = xasprintf("pmd_%u", ovsthread_id_self()); - set_subprogram_name("%s", f->name); poll_cnt = 0; poll_list = NULL; pmd_thread_setaffinity_cpu(f->id); reload: poll_cnt = pmd_load_queues(f, &poll_list, poll_cnt); - atomic_read(&f->change_seq, &port_seq); for (;;) { - unsigned int c_port_seq; int i; for (i = 0; i < poll_cnt; i++) { @@ -1888,14 +1726,15 @@ reload: } if (lc++ > 1024) { - ovsrcu_quiesce(); + unsigned int seq; - /* TODO: need completely userspace based signaling method. - * to keep this thread entirely in userspace. - * For now using atomic counter. */ lc = 0; - atomic_read_explicit(&f->change_seq, &c_port_seq, memory_order_consume); - if (c_port_seq != port_seq) { + + ovsrcu_quiesce(); + + atomic_read_relaxed(&f->change_seq, &seq); + if (seq != port_seq) { + port_seq = seq; break; } } @@ -1910,10 +1749,39 @@ reload: } free(poll_list); - free(f->name); return NULL; } +static void +dp_netdev_disable_upcall(struct dp_netdev *dp) + OVS_ACQUIRES(dp->upcall_rwlock) +{ + fat_rwlock_wrlock(&dp->upcall_rwlock); +} + +static void +dpif_netdev_disable_upcall(struct dpif *dpif) + OVS_NO_THREAD_SAFETY_ANALYSIS +{ + struct dp_netdev *dp = get_dp_netdev(dpif); + dp_netdev_disable_upcall(dp); +} + +static void +dp_netdev_enable_upcall(struct dp_netdev *dp) + OVS_RELEASES(dp->upcall_rwlock) +{ + fat_rwlock_unlock(&dp->upcall_rwlock); +} + +static void +dpif_netdev_enable_upcall(struct dpif *dpif) + OVS_NO_THREAD_SAFETY_ANALYSIS +{ + struct dp_netdev *dp = get_dp_netdev(dpif); + dp_netdev_enable_upcall(dp); +} + static void dp_netdev_set_pmd_threads(struct dp_netdev *dp, int n) { @@ -1943,11 +1811,11 @@ dp_netdev_set_pmd_threads(struct dp_netdev *dp, int n) f->dp = dp; f->id = i; - atomic_store(&f->change_seq, 1); + atomic_init(&f->change_seq, PMD_INITIAL_SEQ); /* Each thread will distribute all devices rx-queues among * themselves. */ - xpthread_create(&f->thread, NULL, pmd_thread_main, f); + f->thread = ovs_thread_create("pmd", pmd_thread_main, f); } } @@ -1962,10 +1830,9 @@ dp_netdev_flow_stats_new_cb(void) static void dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow, - const struct ofpbuf *packet, - const struct flow *key) + int cnt, int size, + uint16_t tcp_flags) { - uint16_t tcp_flags = ntohs(key->tcp_flags); long long int now = time_msec(); struct dp_netdev_flow_stats *bucket; @@ -1974,8 +1841,8 @@ dp_netdev_flow_used(struct dp_netdev_flow *netdev_flow, ovs_mutex_lock(&bucket->mutex); bucket->used = MAX(now, bucket->used); - bucket->packet_count++; - bucket->byte_count += ofpbuf_size(packet); + bucket->packet_count += cnt; + bucket->byte_count += size; bucket->tcp_flags |= tcp_flags; ovs_mutex_unlock(&bucket->mutex); } @@ -1989,182 +1856,385 @@ dp_netdev_stats_new_cb(void) } static void -dp_netdev_count_packet(struct dp_netdev *dp, enum dp_stat_type type) +dp_netdev_count_packet(struct dp_netdev *dp, enum dp_stat_type type, int cnt) { struct dp_netdev_stats *bucket; bucket = ovsthread_stats_bucket_get(&dp->stats, dp_netdev_stats_new_cb); ovs_mutex_lock(&bucket->mutex); - bucket->n[type]++; + bucket->n[type] += cnt; ovs_mutex_unlock(&bucket->mutex); } -static void -dp_netdev_input(struct dp_netdev *dp, struct ofpbuf *packet, - struct pkt_metadata *md) - OVS_REQ_RDLOCK(dp->port_rwlock) +static int +dp_netdev_upcall(struct dp_netdev *dp, struct dpif_packet *packet_, + struct flow *flow, struct flow_wildcards *wc, + enum dpif_upcall_type type, const struct nlattr *userdata, + struct ofpbuf *actions, struct ofpbuf *put_actions) { - struct dp_netdev_flow *netdev_flow; - struct flow key; + struct ofpbuf *packet = &packet_->ofpbuf; - if (ofpbuf_size(packet) < ETH_HEADER_LEN) { - ofpbuf_delete(packet); - return; + if (type == DPIF_UC_MISS) { + dp_netdev_count_packet(dp, DP_STAT_MISS, 1); } - flow_extract(packet, md, &key); - netdev_flow = dp_netdev_lookup_flow(dp, &key); - if (netdev_flow) { - struct dp_netdev_actions *actions; - dp_netdev_flow_used(netdev_flow, packet, &key); + if (OVS_UNLIKELY(!dp->upcall_cb)) { + return ENODEV; + } + + if (OVS_UNLIKELY(!VLOG_DROP_DBG(&upcall_rl))) { + struct ds ds = DS_EMPTY_INITIALIZER; + struct ofpbuf key; + char *packet_str; + + ofpbuf_init(&key, 0); + odp_flow_key_from_flow(&key, flow, &wc->masks, flow->in_port.odp_port, + true); + + packet_str = ofp_packet_to_string(ofpbuf_data(packet), + ofpbuf_size(packet)); - actions = dp_netdev_flow_get_actions(netdev_flow); - dp_netdev_execute_actions(dp, &key, packet, true, md, - actions->actions, actions->size); - dp_netdev_count_packet(dp, DP_STAT_HIT); - } else if (dp->handler_queues) { - dp_netdev_count_packet(dp, DP_STAT_MISS); - dp_netdev_output_userspace(dp, packet, - flow_hash_5tuple(&key, 0) % dp->n_handlers, - DPIF_UC_MISS, &key, NULL); - ofpbuf_delete(packet); + odp_flow_key_format(ofpbuf_data(&key), ofpbuf_size(&key), &ds); + + VLOG_DBG("%s: %s upcall:\n%s\n%s", dp->name, + dpif_upcall_type_to_string(type), ds_cstr(&ds), packet_str); + + ofpbuf_uninit(&key); + free(packet_str); + ds_destroy(&ds); } + + return dp->upcall_cb(packet, flow, type, userdata, actions, wc, + put_actions, dp->upcall_aux); } -static void -dp_netdev_port_input(struct dp_netdev *dp, struct ofpbuf *packet, - struct pkt_metadata *md) - OVS_REQ_RDLOCK(dp->port_rwlock) +struct packet_batch { + unsigned int packet_count; + unsigned int byte_count; + uint16_t tcp_flags; + + struct dp_netdev_flow *flow; + + struct dpif_packet *packets[NETDEV_MAX_RX_BATCH]; + struct pkt_metadata md; +}; + +static inline void +packet_batch_update(struct packet_batch *batch, + struct dpif_packet *packet, const struct miniflow *mf) { - uint32_t *recirc_depth = recirc_depth_get(); + batch->tcp_flags |= miniflow_get_tcp_flags(mf); + batch->packets[batch->packet_count++] = packet; + batch->byte_count += ofpbuf_size(&packet->ofpbuf); +} - *recirc_depth = 0; - dp_netdev_input(dp, packet, md); +static inline void +packet_batch_init(struct packet_batch *batch, struct dp_netdev_flow *flow, + struct pkt_metadata *md) +{ + batch->flow = flow; + batch->md = *md; + + batch->packet_count = 0; + batch->byte_count = 0; + batch->tcp_flags = 0; } -static int -dp_netdev_output_userspace(struct dp_netdev *dp, struct ofpbuf *packet, - int queue_no, int type, const struct flow *flow, - const struct nlattr *userdata) +static inline void +packet_batch_execute(struct packet_batch *batch, struct dp_netdev *dp) { - struct dp_netdev_queue *q; - int error; + struct dp_netdev_actions *actions; + struct dp_netdev_flow *flow = batch->flow; + + dp_netdev_flow_used(batch->flow, batch->packet_count, batch->byte_count, + batch->tcp_flags); + + actions = dp_netdev_flow_get_actions(flow); + + dp_netdev_execute_actions(dp, batch->packets, + batch->packet_count, true, &batch->md, + actions->actions, actions->size); - fat_rwlock_rdlock(&dp->queue_rwlock); - q = &dp->handler_queues[queue_no]; - ovs_mutex_lock(&q->mutex); - if (q->head - q->tail < MAX_QUEUE_LEN) { - struct dp_netdev_upcall *u = &q->upcalls[q->head++ & QUEUE_MASK]; - struct dpif_upcall *upcall = &u->upcall; - struct ofpbuf *buf = &u->buf; - size_t buf_size; - - upcall->type = type; - - /* Allocate buffer big enough for everything. */ - buf_size = ODPUTIL_FLOW_KEY_BYTES; - if (userdata) { - buf_size += NLA_ALIGN(userdata->nla_len); + dp_netdev_count_packet(dp, DP_STAT_HIT, batch->packet_count); +} + +static void +dp_netdev_input(struct dp_netdev *dp, struct dpif_packet **packets, int cnt, + struct pkt_metadata *md) +{ + struct packet_batch batches[NETDEV_MAX_RX_BATCH]; + struct netdev_flow_key keys[NETDEV_MAX_RX_BATCH]; + const struct miniflow *mfs[NETDEV_MAX_RX_BATCH]; /* NULL at bad packets. */ + struct cls_rule *rules[NETDEV_MAX_RX_BATCH]; + size_t n_batches, i; + bool any_miss; + + for (i = 0; i < cnt; i++) { + if (OVS_UNLIKELY(ofpbuf_size(&packets[i]->ofpbuf) < ETH_HEADER_LEN)) { + dpif_packet_delete(packets[i]); + mfs[i] = NULL; + continue; } - buf_size += ofpbuf_size(packet); - ofpbuf_init(buf, buf_size); - - /* Put ODP flow. */ - odp_flow_key_from_flow(buf, flow, flow->in_port.odp_port); - upcall->key = ofpbuf_data(buf); - upcall->key_len = ofpbuf_size(buf); - - /* Put userdata. */ - if (userdata) { - upcall->userdata = ofpbuf_put(buf, userdata, - NLA_ALIGN(userdata->nla_len)); + + miniflow_initialize(&keys[i].flow, keys[i].buf); + miniflow_extract(&packets[i]->ofpbuf, md, &keys[i].flow); + mfs[i] = &keys[i].flow; + } + + any_miss = !classifier_lookup_miniflow_batch(&dp->cls, mfs, rules, cnt); + if (OVS_UNLIKELY(any_miss) && !fat_rwlock_tryrdlock(&dp->upcall_rwlock)) { + uint64_t actions_stub[512 / 8], slow_stub[512 / 8]; + struct ofpbuf actions, put_actions; + struct match match; + + ofpbuf_use_stub(&actions, actions_stub, sizeof actions_stub); + ofpbuf_use_stub(&put_actions, slow_stub, sizeof slow_stub); + + for (i = 0; i < cnt; i++) { + const struct dp_netdev_flow *netdev_flow; + struct ofpbuf *add_actions; + int error; + + if (OVS_LIKELY(rules[i] || !mfs[i])) { + continue; + } + + /* It's possible that an earlier slow path execution installed + * the rule this flow needs. In this case, it's a lot cheaper + * to catch it here than execute a miss. */ + netdev_flow = dp_netdev_lookup_flow(dp, mfs[i]); + if (netdev_flow) { + rules[i] = CONST_CAST(struct cls_rule *, &netdev_flow->cr); + continue; + } + + miniflow_expand(mfs[i], &match.flow); + + ofpbuf_clear(&actions); + ofpbuf_clear(&put_actions); + + error = dp_netdev_upcall(dp, packets[i], &match.flow, &match.wc, + DPIF_UC_MISS, NULL, &actions, + &put_actions); + if (OVS_UNLIKELY(error && error != ENOSPC)) { + continue; + } + + /* We can't allow the packet batching in the next loop to execute + * the actions. Otherwise, if there are any slow path actions, + * we'll send the packet up twice. */ + dp_netdev_execute_actions(dp, &packets[i], 1, false, md, + ofpbuf_data(&actions), + ofpbuf_size(&actions)); + + add_actions = ofpbuf_size(&put_actions) + ? &put_actions + : &actions; + + ovs_mutex_lock(&dp->flow_mutex); + /* XXX: There's a brief race where this flow could have already + * been installed since we last did the flow lookup. This could be + * solved by moving the mutex lock outside the loop, but that's an + * awful long time to be locking everyone out of making flow + * installs. If we move to a per-core classifier, it would be + * reasonable. */ + if (OVS_LIKELY(error != ENOSPC) + && !dp_netdev_lookup_flow(dp, mfs[i])) { + dp_netdev_flow_add(dp, &match, ofpbuf_data(add_actions), + ofpbuf_size(add_actions)); + } + ovs_mutex_unlock(&dp->flow_mutex); } - ofpbuf_set_data(&upcall->packet, - ofpbuf_put(buf, ofpbuf_data(packet), ofpbuf_size(packet))); - ofpbuf_set_size(&upcall->packet, ofpbuf_size(packet)); + ofpbuf_uninit(&actions); + ofpbuf_uninit(&put_actions); + fat_rwlock_unlock(&dp->upcall_rwlock); + } - seq_change(q->seq); + n_batches = 0; + for (i = 0; i < cnt; i++) { + struct dp_netdev_flow *flow; + struct packet_batch *batch; + size_t j; - error = 0; - } else { - dp_netdev_count_packet(dp, DP_STAT_LOST); - error = ENOBUFS; + if (OVS_UNLIKELY(!rules[i] || !mfs[i])) { + continue; + } + + /* XXX: This O(n^2) algortihm makes sense if we're operating under the + * assumption that the number of distinct flows (and therefore the + * number of distinct batches) is quite small. If this turns out not + * to be the case, it may make sense to pre sort based on the + * netdev_flow pointer. That done we can get the appropriate batching + * in O(n * log(n)) instead. */ + batch = NULL; + flow = dp_netdev_flow_cast(rules[i]); + for (j = 0; j < n_batches; j++) { + if (batches[j].flow == flow) { + batch = &batches[j]; + break; + } + } + + if (!batch) { + batch = &batches[n_batches++]; + packet_batch_init(batch, flow, md); + } + packet_batch_update(batch, packets[i], mfs[i]); } - ovs_mutex_unlock(&q->mutex); - fat_rwlock_unlock(&dp->queue_rwlock); - return error; + for (i = 0; i < n_batches; i++) { + packet_batch_execute(&batches[i], dp); + } +} + +static void +dp_netdev_port_input(struct dp_netdev *dp, struct dpif_packet **packets, + int cnt, odp_port_t port_no) +{ + uint32_t *recirc_depth = recirc_depth_get(); + struct pkt_metadata md = PKT_METADATA_INITIALIZER(port_no); + + *recirc_depth = 0; + dp_netdev_input(dp, packets, cnt, &md); } struct dp_netdev_execute_aux { struct dp_netdev *dp; - const struct flow *key; }; static void -dp_execute_cb(void *aux_, struct ofpbuf *packet, +dpif_netdev_register_upcall_cb(struct dpif *dpif, upcall_callback *cb, + void *aux) +{ + struct dp_netdev *dp = get_dp_netdev(dpif); + dp->upcall_aux = aux; + dp->upcall_cb = cb; +} + +static void +dp_execute_cb(void *aux_, struct dpif_packet **packets, int cnt, struct pkt_metadata *md, const struct nlattr *a, bool may_steal) OVS_NO_THREAD_SAFETY_ANALYSIS { struct dp_netdev_execute_aux *aux = aux_; + uint32_t *depth = recirc_depth_get(); + struct dp_netdev *dp = aux->dp; int type = nl_attr_type(a); struct dp_netdev_port *p; - uint32_t *depth = recirc_depth_get(); + int i; switch ((enum ovs_action_attr)type) { case OVS_ACTION_ATTR_OUTPUT: - p = dp_netdev_lookup_port(aux->dp, u32_to_odp(nl_attr_get_u32(a))); - if (p) { - netdev_send(p->netdev, packet, may_steal); + p = dp_netdev_lookup_port(dp, u32_to_odp(nl_attr_get_u32(a))); + if (OVS_LIKELY(p)) { + netdev_send(p->netdev, packets, cnt, may_steal); + } else if (may_steal) { + for (i = 0; i < cnt; i++) { + dpif_packet_delete(packets[i]); + } } break; - case OVS_ACTION_ATTR_USERSPACE: { - const struct nlattr *userdata; + case OVS_ACTION_ATTR_USERSPACE: + if (!fat_rwlock_tryrdlock(&dp->upcall_rwlock)) { + const struct nlattr *userdata; + struct ofpbuf actions; + struct flow flow; + + userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA); + ofpbuf_init(&actions, 0); + + for (i = 0; i < cnt; i++) { + int error; - userdata = nl_attr_find_nested(a, OVS_USERSPACE_ATTR_USERDATA); + ofpbuf_clear(&actions); - dp_netdev_output_userspace(aux->dp, packet, - flow_hash_5tuple(aux->key, 0) - % aux->dp->n_handlers, - DPIF_UC_ACTION, aux->key, - userdata); + flow_extract(&packets[i]->ofpbuf, md, &flow); + error = dp_netdev_upcall(dp, packets[i], &flow, NULL, + DPIF_UC_ACTION, userdata, &actions, + NULL); + if (!error || error == ENOSPC) { + dp_netdev_execute_actions(dp, &packets[i], 1, false, md, + ofpbuf_data(&actions), + ofpbuf_size(&actions)); + } - if (may_steal) { - ofpbuf_delete(packet); + if (may_steal) { + dpif_packet_delete(packets[i]); + } + } + ofpbuf_uninit(&actions); + fat_rwlock_unlock(&dp->upcall_rwlock); + } + + break; + + case OVS_ACTION_ATTR_HASH: { + const struct ovs_action_hash *hash_act; + struct netdev_flow_key key; + uint32_t hash; + + hash_act = nl_attr_get(a); + + miniflow_initialize(&key.flow, key.buf); + + for (i = 0; i < cnt; i++) { + + /* XXX: this is slow. Use RSS hash in the future */ + miniflow_extract(&packets[i]->ofpbuf, md, &key.flow); + + if (hash_act->hash_alg == OVS_HASH_ALG_L4) { + /* Hash need not be symmetric, nor does it need to include + * L2 fields. */ + hash = miniflow_hash_5tuple(&key.flow, hash_act->hash_basis); + } else { + VLOG_WARN("Unknown hash algorithm specified " + "for the hash action."); + hash = 2; + } + + if (!hash) { + hash = 1; /* 0 is not valid */ + } + + if (i == 0) { + md->dp_hash = hash; + } + packets[i]->dp_hash = hash; } break; } case OVS_ACTION_ATTR_RECIRC: if (*depth < MAX_RECIRC_DEPTH) { - struct pkt_metadata recirc_md = *md; - struct ofpbuf *recirc_packet; - const struct ovs_action_recirc *act; - recirc_packet = may_steal ? packet : ofpbuf_clone(packet); + (*depth)++; + for (i = 0; i < cnt; i++) { + struct dpif_packet *recirc_pkt; + struct pkt_metadata recirc_md = *md; - act = nl_attr_get(a); - recirc_md.recirc_id = act->recirc_id; - recirc_md.dp_hash = 0; + recirc_pkt = (may_steal) ? packets[i] + : dpif_packet_clone(packets[i]); - if (act->hash_alg == OVS_RECIRC_HASH_ALG_L4) { - recirc_md.dp_hash = flow_hash_symmetric_l4(aux->key, - act->hash_bias); - if (!recirc_md.dp_hash) { - recirc_md.dp_hash = 1; /* 0 is not valid */ - } - } + recirc_md.recirc_id = nl_attr_get_u32(a); - (*depth)++; - dp_netdev_input(aux->dp, recirc_packet, &recirc_md); + /* Hash is private to each packet */ + recirc_md.dp_hash = dpif_packet_get_dp_hash(packets[i]); + + dp_netdev_input(dp, &recirc_pkt, 1, &recirc_md); + } (*depth)--; break; } else { VLOG_WARN("Packet dropped. Max recirculation depth exceeded."); + if (may_steal) { + for (i = 0; i < cnt; i++) { + dpif_packet_delete(packets[i]); + } + } } break; @@ -2181,15 +2251,15 @@ dp_execute_cb(void *aux_, struct ofpbuf *packet, } static void -dp_netdev_execute_actions(struct dp_netdev *dp, const struct flow *key, - struct ofpbuf *packet, bool may_steal, - struct pkt_metadata *md, +dp_netdev_execute_actions(struct dp_netdev *dp, + struct dpif_packet **packets, int cnt, + bool may_steal, struct pkt_metadata *md, const struct nlattr *actions, size_t actions_len) { - struct dp_netdev_execute_aux aux = {dp, key}; + struct dp_netdev_execute_aux aux = {dp}; - odp_execute_actions(&aux, packet, may_steal, md, - actions, actions_len, dp_execute_cb); + odp_execute_actions(&aux, packets, cnt, may_steal, md, actions, + actions_len, dp_execute_cb); } const struct dpif_class dpif_netdev_class = { @@ -2212,31 +2282,30 @@ const struct dpif_class dpif_netdev_class = { dpif_netdev_port_dump_done, dpif_netdev_port_poll, dpif_netdev_port_poll_wait, - dpif_netdev_flow_get, - dpif_netdev_flow_put, - dpif_netdev_flow_del, dpif_netdev_flow_flush, - dpif_netdev_flow_dump_state_init, - dpif_netdev_flow_dump_start, + dpif_netdev_flow_dump_create, + dpif_netdev_flow_dump_destroy, + dpif_netdev_flow_dump_thread_create, + dpif_netdev_flow_dump_thread_destroy, dpif_netdev_flow_dump_next, - NULL, - dpif_netdev_flow_dump_done, - dpif_netdev_flow_dump_state_uninit, - dpif_netdev_execute, - NULL, /* operate */ - dpif_netdev_recv_set, - dpif_netdev_handlers_set, + dpif_netdev_operate, + NULL, /* recv_set */ + NULL, /* handlers_set */ dpif_netdev_queue_to_priority, - dpif_netdev_recv, - dpif_netdev_recv_wait, - dpif_netdev_recv_purge, + NULL, /* recv */ + NULL, /* recv_wait */ + NULL, /* recv_purge */ + dpif_netdev_register_upcall_cb, + dpif_netdev_enable_upcall, + dpif_netdev_disable_upcall, }; static void dpif_dummy_change_port_number(struct unixctl_conn *conn, int argc OVS_UNUSED, const char *argv[], void *aux OVS_UNUSED) { - struct dp_netdev_port *port; + struct dp_netdev_port *old_port; + struct dp_netdev_port *new_port; struct dp_netdev *dp; odp_port_t port_no; @@ -2250,8 +2319,8 @@ dpif_dummy_change_port_number(struct unixctl_conn *conn, int argc OVS_UNUSED, ovs_refcount_ref(&dp->ref_cnt); ovs_mutex_unlock(&dp_netdev_mutex); - ovs_rwlock_wrlock(&dp->port_rwlock); - if (get_port_by_name(dp, argv[2], &port)) { + ovs_mutex_lock(&dp->port_mutex); + if (get_port_by_name(dp, argv[2], &old_port)) { unixctl_command_reply_error(conn, "unknown port"); goto exit; } @@ -2265,14 +2334,52 @@ dpif_dummy_change_port_number(struct unixctl_conn *conn, int argc OVS_UNUSED, unixctl_command_reply_error(conn, "port number already in use"); goto exit; } - hmap_remove(&dp->ports, &port->node); - port->port_no = port_no; - hmap_insert(&dp->ports, &port->node, hash_int(odp_to_u32(port_no), 0)); + + /* Remove old port. */ + cmap_remove(&dp->ports, &old_port->node, hash_port_no(old_port->port_no)); + ovsrcu_postpone(free, old_port); + + /* Insert new port (cmap semantics mean we cannot re-insert 'old_port'). */ + new_port = xmemdup(old_port, sizeof *old_port); + new_port->port_no = port_no; + cmap_insert(&dp->ports, &new_port->node, hash_port_no(port_no)); + seq_change(dp->port_seq); unixctl_command_reply(conn, NULL); exit: - ovs_rwlock_unlock(&dp->port_rwlock); + ovs_mutex_unlock(&dp->port_mutex); + dp_netdev_unref(dp); +} + +static void +dpif_dummy_delete_port(struct unixctl_conn *conn, int argc OVS_UNUSED, + const char *argv[], void *aux OVS_UNUSED) +{ + struct dp_netdev_port *port; + struct dp_netdev *dp; + + ovs_mutex_lock(&dp_netdev_mutex); + dp = shash_find_data(&dp_netdevs, argv[1]); + if (!dp || !dpif_netdev_class_is_dummy(dp->class)) { + ovs_mutex_unlock(&dp_netdev_mutex); + unixctl_command_reply_error(conn, "unknown datapath or not a dummy"); + return; + } + ovs_refcount_ref(&dp->ref_cnt); + ovs_mutex_unlock(&dp_netdev_mutex); + + ovs_mutex_lock(&dp->port_mutex); + if (get_port_by_name(dp, argv[2], &port)) { + unixctl_command_reply_error(conn, "unknown port"); + } else if (port->port_no == ODPP_LOCAL) { + unixctl_command_reply_error(conn, "can't delete local port"); + } else { + do_del_port(dp, port); + unixctl_command_reply(conn, NULL); + } + ovs_mutex_unlock(&dp->port_mutex); + dp_netdev_unref(dp); } @@ -2309,4 +2416,6 @@ dpif_dummy_register(bool override) unixctl_command_register("dpif-dummy/change-port-number", "DP PORT NEW-NUMBER", 3, 3, dpif_dummy_change_port_number, NULL); + unixctl_command_register("dpif-dummy/delete-port", "DP PORT", + 2, 2, dpif_dummy_delete_port, NULL); }