ofproto: per-table statistics
[cascardo/ovs.git] / ofproto / ofproto.c
index a517264..e9bd24b 100644 (file)
@@ -58,6 +58,7 @@
 #include "unaligned.h"
 #include "unixctl.h"
 #include "vlog.h"
+#include "bundles.h"
 
 VLOG_DEFINE_THIS_MODULE(ofproto);
 
@@ -68,6 +69,11 @@ COVERAGE_DEFINE(ofproto_recv_openflow);
 COVERAGE_DEFINE(ofproto_reinit_ports);
 COVERAGE_DEFINE(ofproto_update_port);
 
+/* Default fields to use for prefix tries in each flow table, unless something
+ * else is configured. */
+const enum mf_field_id default_prefix_fields[2] =
+    { MFF_IPV4_DST, MFF_IPV4_SRC };
+
 enum ofproto_state {
     S_OPENFLOW,                 /* Processing OpenFlow commands. */
     S_EVICT,                    /* Evicting flows from over-limit tables. */
@@ -125,7 +131,7 @@ struct ofoperation {
 
     /* OFOPERATION_MODIFY, OFOPERATION_REPLACE: The old actions, if the actions
      * are changing. */
-    struct rule_actions *actions;
+    const struct rule_actions *actions;
 
     /* OFOPERATION_DELETE. */
     enum ofp_flow_removed_reason reason; /* Reason flow was removed. */
@@ -259,7 +265,6 @@ struct ofport_usage {
 };
 
 /* rule. */
-static void ofproto_rule_destroy__(struct rule *);
 static void ofproto_rule_send_removed(struct rule *, uint8_t reason);
 static bool rule_is_modifiable(const struct rule *rule,
                                enum ofputil_flow_mod_flags flag);
@@ -268,6 +273,9 @@ static bool rule_is_modifiable(const struct rule *rule,
 static enum ofperr add_flow(struct ofproto *, struct ofconn *,
                             struct ofputil_flow_mod *,
                             const struct ofp_header *);
+static void do_add_flow(struct ofproto *, struct ofconn *,
+                        const struct ofp_header *request, uint32_t buffer_id,
+                        struct rule *);
 static enum ofperr modify_flows__(struct ofproto *, struct ofconn *,
                                   struct ofputil_flow_mod *,
                                   const struct ofp_header *,
@@ -527,10 +535,6 @@ ofproto_create(const char *datapath_name, const char *datapath_type,
     ofproto->n_pending = 0;
     hmap_init(&ofproto->deletions);
     guarded_list_init(&ofproto->rule_executes);
-    ofproto->n_add = ofproto->n_delete = ofproto->n_modify = 0;
-    ofproto->first_op = ofproto->last_op = LLONG_MIN;
-    ofproto->next_op_report = LLONG_MAX;
-    ofproto->op_backoff = LLONG_MIN;
     ofproto->vlan_bitmap = NULL;
     ofproto->vlans_changed = false;
     ofproto->min_mtu = INT_MAX;
@@ -1016,10 +1020,12 @@ ofproto_port_set_bfd(struct ofproto *ofproto, ofp_port_t ofp_port,
     }
 }
 
-/* Populates 'status' with key value pairs indicating the status of the BFD
- * session on 'ofp_port'.  This information is intended to be populated in the
- * OVS database.  Has no effect if 'ofp_port' is not na OpenFlow port in
- * 'ofproto'. */
+/* Populates 'status' with the status of BFD on 'ofport'.  Returns 0 on
+ * success.  Returns a negative number if there is no status change since
+ * last update.  Returns a positive errno otherwise.  Has no effect if
+ * 'ofp_port' is not an OpenFlow port in 'ofproto'.
+ *
+ * The caller must provide and own '*status'. */
 int
 ofproto_port_get_bfd_status(struct ofproto *ofproto, ofp_port_t ofp_port,
                             struct smap *status)
@@ -1307,9 +1313,9 @@ ofproto_destroy__(struct ofproto *ofproto)
     ovs_assert(list_is_empty(&ofproto->pending));
 
     destroy_rule_executes(ofproto);
-    guarded_list_destroy(&ofproto->rule_executes);
-
     delete_group(ofproto, OFPG_ALL);
+
+    guarded_list_destroy(&ofproto->rule_executes);
     ovs_rwlock_destroy(&ofproto->groups_rwlock);
     hmap_destroy(&ofproto->groups);
 
@@ -1369,7 +1375,8 @@ ofproto_destroy(struct ofproto *p)
     }
 
     p->ofproto_class->destruct(p);
-    ofproto_destroy__(p);
+    /* Destroying rules is deferred, must have 'ofproto' around for them. */
+    ovsrcu_postpone(ofproto_destroy__, p);
 }
 
 /* Destroys the datapath with the respective 'name' and 'type'.  With the Linux
@@ -1514,7 +1521,13 @@ ofproto_run(struct ofproto *p)
          * need this two-phase approach. */
         sset_init(&devnames);
         HMAP_FOR_EACH (ofport, hmap_node, &p->ports) {
-            sset_add(&devnames, netdev_get_name(ofport->netdev));
+            uint64_t port_change_seq;
+
+            port_change_seq = netdev_get_change_seq(ofport->netdev);
+            if (ofport->change_seq != port_change_seq) {
+                ofport->change_seq = port_change_seq;
+                sset_add(&devnames, netdev_get_name(ofport->netdev));
+            }
         }
         SSET_FOR_EACH (devname, &devnames) {
             update_port(p, devname);
@@ -1550,43 +1563,6 @@ ofproto_run(struct ofproto *p)
         OVS_NOT_REACHED();
     }
 
-    if (time_msec() >= p->next_op_report) {
-        long long int ago = (time_msec() - p->first_op) / 1000;
-        long long int interval = (p->last_op - p->first_op) / 1000;
-        struct ds s;
-
-        ds_init(&s);
-        ds_put_format(&s, "%d flow_mods ",
-                      p->n_add + p->n_delete + p->n_modify);
-        if (interval == ago) {
-            ds_put_format(&s, "in the last %lld s", ago);
-        } else if (interval) {
-            ds_put_format(&s, "in the %lld s starting %lld s ago",
-                          interval, ago);
-        } else {
-            ds_put_format(&s, "%lld s ago", ago);
-        }
-
-        ds_put_cstr(&s, " (");
-        if (p->n_add) {
-            ds_put_format(&s, "%d adds, ", p->n_add);
-        }
-        if (p->n_delete) {
-            ds_put_format(&s, "%d deletes, ", p->n_delete);
-        }
-        if (p->n_modify) {
-            ds_put_format(&s, "%d modifications, ", p->n_modify);
-        }
-        s.length -= 2;
-        ds_put_char(&s, ')');
-
-        VLOG_INFO("%s: %s", p->name, ds_cstr(&s));
-        ds_destroy(&s);
-
-        p->n_add = p->n_delete = p->n_modify = 0;
-        p->next_op_report = LLONG_MAX;
-    }
-
     return error;
 }
 
@@ -1930,7 +1906,7 @@ ofproto_add_flow(struct ofproto *ofproto, const struct match *match,
     rule = rule_from_cls_rule(classifier_find_match_exactly(
                                   &ofproto->tables[0].cls, match, priority));
     if (rule) {
-        struct rule_actions *actions = rule_get_actions(rule);
+        const struct rule_actions *actions = rule_get_actions(rule);
         must_add = !ofpacts_equal(actions->ofpacts, actions->ofpacts_len,
                                   ofpacts, ofpacts_len);
     } else {
@@ -1964,14 +1940,13 @@ ofproto_flow_mod(struct ofproto *ofproto, struct ofputil_flow_mod *fm)
     if (fm->command == OFPFC_MODIFY_STRICT && fm->table_id != OFPTT_ALL
         && !(fm->flags & OFPUTIL_FF_RESET_COUNTS)) {
         struct oftable *table = &ofproto->tables[fm->table_id];
-        struct cls_rule match_rule;
         struct rule *rule;
         bool done = false;
 
-        cls_rule_init(&match_rule, &fm->match, fm->priority);
         fat_rwlock_rdlock(&table->cls.rwlock);
-        rule = rule_from_cls_rule(classifier_find_rule_exactly(&table->cls,
-                                                               &match_rule));
+        rule = rule_from_cls_rule(classifier_find_match_exactly(&table->cls,
+                                                                &fm->match,
+                                                                fm->priority));
         if (rule) {
             /* Reading many of the rule fields and writing on 'modified'
              * requires the rule->mutex.  Also, rule->actions may change
@@ -2003,6 +1978,47 @@ ofproto_flow_mod(struct ofproto *ofproto, struct ofputil_flow_mod *fm)
     return handle_flow_mod__(ofproto, NULL, fm, NULL);
 }
 
+/* Resets the modified time for 'rule' or an equivalent rule. If 'rule' is not
+ * in the classifier, but an equivalent rule is, unref 'rule' and ref the new
+ * rule. Otherwise if 'rule' is no longer installed in the classifier,
+ * reinstall it.
+ *
+ * Returns the rule whose modified time has been reset. */
+struct rule *
+ofproto_refresh_rule(struct rule *rule)
+{
+    const struct oftable *table = &rule->ofproto->tables[rule->table_id];
+    const struct cls_rule *cr = &rule->cr;
+    struct rule *r;
+
+    /* do_add_flow() requires that the rule is not installed. We lock the
+     * ofproto_mutex here so that another thread cannot add the flow before
+     * we get a chance to add it.*/
+    ovs_mutex_lock(&ofproto_mutex);
+
+    fat_rwlock_rdlock(&table->cls.rwlock);
+    r = rule_from_cls_rule(classifier_find_rule_exactly(&table->cls, cr));
+    if (r != rule) {
+        ofproto_rule_ref(r);
+    }
+    fat_rwlock_unlock(&table->cls.rwlock);
+
+    if (!r) {
+        do_add_flow(rule->ofproto, NULL, NULL, 0, rule);
+    } else if  (r != rule) {
+        ofproto_rule_unref(rule);
+        rule = r;
+    }
+    ovs_mutex_unlock(&ofproto_mutex);
+
+    /* Refresh the modified time for the rule. */
+    ovs_mutex_lock(&rule->mutex);
+    rule->modified = MAX(rule->modified, time_msec());
+    ovs_mutex_unlock(&rule->mutex);
+
+    return rule;
+}
+
 /* Searches for a rule with matching criteria exactly equal to 'target' in
  * ofproto's table 0 and, if it finds one, deletes it.
  *
@@ -2131,7 +2147,7 @@ dealloc_ofp_port(struct ofproto *ofproto, ofp_port_t ofp_port)
 
 /* Opens and returns a netdev for 'ofproto_port' in 'ofproto', or a null
  * pointer if the netdev cannot be opened.  On success, also fills in
- * 'opp'.  */
+ * '*pp'.  */
 static struct netdev *
 ofport_open(struct ofproto *ofproto,
             struct ofproto_port *ofproto_port,
@@ -2174,7 +2190,7 @@ ofport_open(struct ofproto *ofproto,
 }
 
 /* Returns true if most fields of 'a' and 'b' are equal.  Differences in name,
- * port number, and 'config' bits other than OFPUTIL_PS_LINK_DOWN are
+ * port number, and 'config' bits other than OFPUTIL_PC_PORT_DOWN are
  * disregarded. */
 static bool
 ofport_equal(const struct ofputil_phy_port *a,
@@ -2210,6 +2226,7 @@ ofport_install(struct ofproto *p,
     }
     ofport->ofproto = p;
     ofport->netdev = netdev;
+    ofport->change_seq = netdev_get_change_seq(netdev);
     ofport->pp = *pp;
     ofport->ofp_port = pp->port_no;
     ofport->created = time_msec();
@@ -2446,6 +2463,7 @@ update_port(struct ofproto *ofproto, const char *name)
              * Don't close the old netdev yet in case port_modified has to
              * remove a retained reference to it.*/
             port->netdev = netdev;
+            port->change_seq = netdev_get_change_seq(netdev);
 
             if (port->ofproto->ofproto_class->port_modified) {
                 port->ofproto->ofproto_class->port_modified(port);
@@ -2590,6 +2608,23 @@ update_mtu(struct ofproto *p, struct ofport *port)
     }
 }
 \f
+static void
+ofproto_rule_destroy__(struct rule *rule)
+    OVS_NO_THREAD_SAFETY_ANALYSIS
+{
+    cls_rule_destroy(CONST_CAST(struct cls_rule *, &rule->cr));
+    rule_actions_destroy(rule_get_actions(rule));
+    ovs_mutex_destroy(&rule->mutex);
+    rule->ofproto->ofproto_class->rule_dealloc(rule);
+}
+
+static void
+rule_destroy_cb(struct rule *rule)
+{
+    rule->ofproto->ofproto_class->rule_destruct(rule);
+    ofproto_rule_destroy__(rule);
+}
+
 void
 ofproto_rule_ref(struct rule *rule)
 {
@@ -2598,60 +2633,65 @@ ofproto_rule_ref(struct rule *rule)
     }
 }
 
+/* Decrements 'rule''s ref_count and schedules 'rule' to be destroyed if the
+ * ref_count reaches 0.
+ *
+ * Use of RCU allows short term use (between RCU quiescent periods) without
+ * keeping a reference.  A reference must be taken if the rule needs to
+ * stay around accross the RCU quiescent periods. */
 void
 ofproto_rule_unref(struct rule *rule)
 {
     if (rule && ovs_refcount_unref(&rule->ref_count) == 1) {
-        rule->ofproto->ofproto_class->rule_destruct(rule);
-        ofproto_rule_destroy__(rule);
+        ovsrcu_postpone(rule_destroy_cb, rule);
     }
 }
 
-static void
-ofproto_rule_destroy__(struct rule *rule)
-    OVS_NO_THREAD_SAFETY_ANALYSIS
+void
+ofproto_group_ref(struct ofgroup *group)
 {
-    cls_rule_destroy(CONST_CAST(struct cls_rule *, &rule->cr));
-    rule_actions_destroy(rule_get_actions(rule));
-    ovs_mutex_destroy(&rule->mutex);
-    rule->ofproto->ofproto_class->rule_dealloc(rule);
+    if (group) {
+        ovs_refcount_ref(&group->ref_count);
+    }
+}
+
+void
+ofproto_group_unref(struct ofgroup *group)
+{
+    if (group && ovs_refcount_unref(&group->ref_count) == 1) {
+        group->ofproto->ofproto_class->group_destruct(group);
+        ofputil_bucket_list_destroy(&group->buckets);
+        group->ofproto->ofproto_class->group_dealloc(group);
+    }
 }
 
 static uint32_t get_provider_meter_id(const struct ofproto *,
                                       uint32_t of_meter_id);
 
-/* Creates and returns a new 'struct rule_actions', with a ref_count of 1,
- * whose actions are a copy of from the 'ofpacts_len' bytes of 'ofpacts'. */
-struct rule_actions *
+/* Creates and returns a new 'struct rule_actions', whose actions are a copy
+ * of from the 'ofpacts_len' bytes of 'ofpacts'. */
+const struct rule_actions *
 rule_actions_create(const struct ofproto *ofproto,
                     const struct ofpact *ofpacts, size_t ofpacts_len)
 {
     struct rule_actions *actions;
 
-    actions = xmalloc(sizeof *actions);
-    actions->ofpacts = xmemdup(ofpacts, ofpacts_len);
+    actions = xmalloc(sizeof *actions + ofpacts_len);
     actions->ofpacts_len = ofpacts_len;
     actions->provider_meter_id
         = get_provider_meter_id(ofproto,
                                 ofpacts_get_meter(ofpacts, ofpacts_len));
+    memcpy(actions->ofpacts, ofpacts, ofpacts_len);
 
     return actions;
 }
 
-static void
-rule_actions_destroy_cb(struct rule_actions *actions)
-{
-    free(actions->ofpacts);
-    free(actions);
-}
-
-/* Decrements 'actions''s ref_count and frees 'actions' if the ref_count
- * reaches 0. */
+/* Free the actions after the RCU quiescent period is reached. */
 void
-rule_actions_destroy(struct rule_actions *actions)
+rule_actions_destroy(const struct rule_actions *actions)
 {
     if (actions) {
-        ovsrcu_postpone(rule_actions_destroy_cb, actions);
+        ovsrcu_postpone(free, CONST_CAST(struct rule_actions *, actions));
     }
 }
 
@@ -3021,7 +3061,7 @@ handle_port_mod(struct ofconn *ofconn, const struct ofp_header *oh)
         return error;
     }
 
-    error = ofputil_decode_port_mod(oh, &pm);
+    error = ofputil_decode_port_mod(oh, &pm, false);
     if (error) {
         return error;
     }
@@ -3153,55 +3193,62 @@ append_port_stat(struct ofport *port, struct list *replies)
     ofputil_append_port_stat(replies, &ops);
 }
 
-static enum ofperr
-handle_port_stats_request(struct ofconn *ofconn,
-                          const struct ofp_header *request)
+static void
+handle_port_request(struct ofconn *ofconn,
+                    const struct ofp_header *request, ofp_port_t port_no,
+                    void (*cb)(struct ofport *, struct list *replies))
 {
-    struct ofproto *p = ofconn_get_ofproto(ofconn);
+    struct ofproto *ofproto = ofconn_get_ofproto(ofconn);
     struct ofport *port;
     struct list replies;
-    ofp_port_t port_no;
-    enum ofperr error;
-
-    error = ofputil_decode_port_stats_request(request, &port_no);
-    if (error) {
-        return error;
-    }
 
     ofpmp_init(&replies, request);
     if (port_no != OFPP_ANY) {
-        port = ofproto_get_port(p, port_no);
+        port = ofproto_get_port(ofproto, port_no);
         if (port) {
-            append_port_stat(port, &replies);
+            cb(port, &replies);
         }
     } else {
-        HMAP_FOR_EACH (port, hmap_node, &p->ports) {
-            append_port_stat(port, &replies);
+        HMAP_FOR_EACH (port, hmap_node, &ofproto->ports) {
+            cb(port, &replies);
         }
     }
 
     ofconn_send_replies(ofconn, &replies);
-    return 0;
+}
+
+static enum ofperr
+handle_port_stats_request(struct ofconn *ofconn,
+                          const struct ofp_header *request)
+{
+    ofp_port_t port_no;
+    enum ofperr error;
+
+    error = ofputil_decode_port_stats_request(request, &port_no);
+    if (!error) {
+        handle_port_request(ofconn, request, port_no, append_port_stat);
+    }
+    return error;
+}
+
+static void
+append_port_desc(struct ofport *port, struct list *replies)
+{
+    ofputil_append_port_desc_stats_reply(&port->pp, replies);
 }
 
 static enum ofperr
 handle_port_desc_stats_request(struct ofconn *ofconn,
                                const struct ofp_header *request)
 {
-    struct ofproto *p = ofconn_get_ofproto(ofconn);
-    enum ofp_version version;
-    struct ofport *port;
-    struct list replies;
-
-    ofpmp_init(&replies, request);
+    ofp_port_t port_no;
+    enum ofperr error;
 
-    version = ofputil_protocol_to_ofp_version(ofconn_get_protocol(ofconn));
-    HMAP_FOR_EACH (port, hmap_node, &p->ports) {
-        ofputil_append_port_desc_stats_reply(version, &port->pp, &replies);
+    error = ofputil_decode_port_desc_stats_request(request, &port_no);
+    if (!error) {
+        handle_port_request(ofconn, request, port_no, append_port_desc);
     }
-
-    ofconn_send_replies(ofconn, &replies);
-    return 0;
+    return error;
 }
 
 static uint32_t
@@ -3595,7 +3642,7 @@ handle_flow_stats_request(struct ofconn *ofconn,
         long long int now = time_msec();
         struct ofputil_flow_stats fs;
         long long int created, used, modified;
-        struct rule_actions *actions;
+        const struct rule_actions *actions;
         enum ofputil_flow_mod_flags flags;
 
         ovs_mutex_lock(&rule->mutex);
@@ -3636,7 +3683,7 @@ static void
 flow_stats_ds(struct rule *rule, struct ds *results)
 {
     uint64_t packet_count, byte_count;
-    struct rule_actions *actions;
+    const struct rule_actions *actions;
     long long int created, used;
 
     rule->ofproto->ofproto_class->rule_get_stats(rule, &packet_count,
@@ -3691,20 +3738,22 @@ ofproto_get_netflow_ids(const struct ofproto *ofproto,
     ofproto->ofproto_class->get_netflow_ids(ofproto, engine_type, engine_id);
 }
 
-/* Checks the status of CFM configured on 'ofp_port' within 'ofproto'.  Returns
- * true if the port's CFM status was successfully stored into '*status'.
- * Returns false if the port did not have CFM configured, in which case
- * '*status' is indeterminate.
+/* Checks the status of CFM configured on 'ofp_port' within 'ofproto'.
+ * Returns 0 if the port's CFM status was successfully stored into
+ * '*status'.  Returns positive errno if the port did not have CFM
+ * configured.  Returns negative number if there is no status change
+ * since last update.
  *
- * The caller must provide and owns '*status', and must free 'status->rmps'. */
-bool
+ * The caller must provide and own '*status', and must free 'status->rmps'.
+ * '*status' is indeterminate if the return value is non-zero. */
+int
 ofproto_port_get_cfm_status(const struct ofproto *ofproto, ofp_port_t ofp_port,
                             struct ofproto_cfm_status *status)
 {
     struct ofport *ofport = ofproto_get_port(ofproto, ofp_port);
-    return (ofport
-            && ofproto->ofproto_class->get_cfm_status
-            && ofproto->ofproto_class->get_cfm_status(ofport, status));
+    return (ofport && ofproto->ofproto_class->get_cfm_status
+            ? ofproto->ofproto_class->get_cfm_status(ofport, status)
+            : EOPNOTSUPP);
 }
 
 static enum ofperr
@@ -3894,7 +3943,8 @@ is_flow_deletion_pending(const struct ofproto *ofproto,
         HMAP_FOR_EACH_WITH_HASH (op, hmap_node,
                                  cls_rule_hash(cls_rule, table_id),
                                  &ofproto->deletions) {
-            if (cls_rule_equal(cls_rule, &op->rule->cr)) {
+            if (op->rule->table_id == table_id
+                && cls_rule_equal(cls_rule, &op->rule->cr)) {
                 return true;
             }
         }
@@ -3951,7 +4001,6 @@ add_flow(struct ofproto *ofproto, struct ofconn *ofconn,
     OVS_REQUIRES(ofproto_mutex)
 {
     struct oftable *table;
-    struct ofopgroup *group;
     struct cls_rule cr;
     struct rule *rule;
     uint8_t table_id;
@@ -4089,14 +4138,25 @@ add_flow(struct ofproto *ofproto, struct ofconn *ofconn,
     }
 
     /* Insert rule. */
+    do_add_flow(ofproto, ofconn, request, fm->buffer_id, rule);
+
+    return error;
+}
+
+static void
+do_add_flow(struct ofproto *ofproto, struct ofconn *ofconn,
+            const struct ofp_header *request, uint32_t buffer_id,
+            struct rule *rule)
+    OVS_REQUIRES(ofproto_mutex)
+{
+    struct ofopgroup *group;
+
     oftable_insert_rule(rule);
 
-    group = ofopgroup_create(ofproto, ofconn, request, fm->buffer_id);
+    group = ofopgroup_create(ofproto, ofconn, request, buffer_id);
     ofoperation_create(group, rule, OFOPERATION_ADD, 0);
     ofproto->ofproto_class->rule_insert(rule);
     ofopgroup_submit(group);
-
-    return error;
 }
 \f
 /* OFPFC_MODIFY and OFPFC_MODIFY_STRICT. */
@@ -4166,7 +4226,7 @@ modify_flows__(struct ofproto *ofproto, struct ofconn *ofconn,
 
         reset_counters = (fm->flags & OFPUTIL_FF_RESET_COUNTS) != 0;
         if (actions_changed || reset_counters) {
-            struct rule_actions *new_actions;
+            const struct rule_actions *new_actions;
 
             op->actions = rule_get_actions(rule);
             new_actions = rule_actions_create(ofproto,
@@ -4443,7 +4503,6 @@ handle_flow_mod(struct ofconn *ofconn, const struct ofp_header *oh)
     uint64_t ofpacts_stub[1024 / 8];
     struct ofpbuf ofpacts;
     enum ofperr error;
-    long long int now;
 
     error = reject_slave_controller(ofconn);
     if (error) {
@@ -4465,31 +4524,7 @@ handle_flow_mod(struct ofconn *ofconn, const struct ofp_header *oh)
         goto exit_free_ofpacts;
     }
 
-    /* Record the operation for logging a summary report. */
-    switch (fm.command) {
-    case OFPFC_ADD:
-        ofproto->n_add++;
-        break;
-
-    case OFPFC_MODIFY:
-    case OFPFC_MODIFY_STRICT:
-        ofproto->n_modify++;
-        break;
-
-    case OFPFC_DELETE:
-    case OFPFC_DELETE_STRICT:
-        ofproto->n_delete++;
-        break;
-    }
-
-    now = time_msec();
-    if (ofproto->next_op_report == LLONG_MAX) {
-        ofproto->first_op = now;
-        ofproto->next_op_report = MAX(now + 10 * 1000,
-                                      ofproto->op_backoff);
-        ofproto->op_backoff = ofproto->next_op_report + 60 * 1000;
-    }
-    ofproto->last_op = now;
+    ofconn_report_flow_mod(ofconn, fm.command);
 
 exit_free_ofpacts:
     ofpbuf_uninit(&ofpacts);
@@ -5317,16 +5352,19 @@ handle_meter_request(struct ofconn *ofconn, const struct ofp_header *request,
     return 0;
 }
 
+/* If the group exists, this function increments the groups's reference count.
+ *
+ * Make sure to call ofproto_group_unref() after no longer needing to maintain
+ * a reference to the group. */
 bool
 ofproto_group_lookup(const struct ofproto *ofproto, uint32_t group_id,
                      struct ofgroup **group)
-    OVS_TRY_RDLOCK(true, (*group)->rwlock)
 {
     ovs_rwlock_rdlock(&ofproto->groups_rwlock);
     HMAP_FOR_EACH_IN_BUCKET (*group, hmap_node,
                              hash_int(group_id, 0), &ofproto->groups) {
         if ((*group)->group_id == group_id) {
-            ovs_rwlock_rdlock(&(*group)->rwlock);
+            ofproto_group_ref(*group);
             ovs_rwlock_unlock(&ofproto->groups_rwlock);
             return true;
         }
@@ -5335,28 +5373,18 @@ ofproto_group_lookup(const struct ofproto *ofproto, uint32_t group_id,
     return false;
 }
 
-void
-ofproto_group_release(struct ofgroup *group)
-    OVS_RELEASES(group->rwlock)
-{
-    ovs_rwlock_unlock(&group->rwlock);
-}
-
 static bool
 ofproto_group_write_lookup(const struct ofproto *ofproto, uint32_t group_id,
                            struct ofgroup **group)
-    OVS_TRY_WRLOCK(true, ofproto->groups_rwlock)
-    OVS_TRY_WRLOCK(true, (*group)->rwlock)
+    OVS_ACQUIRES(ofproto->groups_rwlock)
 {
     ovs_rwlock_wrlock(&ofproto->groups_rwlock);
     HMAP_FOR_EACH_IN_BUCKET (*group, hmap_node,
                              hash_int(group_id, 0), &ofproto->groups) {
         if ((*group)->group_id == group_id) {
-            ovs_rwlock_wrlock(&(*group)->rwlock);
             return true;
         }
     }
-    ovs_rwlock_unlock(&ofproto->groups_rwlock);
     return false;
 }
 
@@ -5392,7 +5420,7 @@ static uint32_t
 group_get_ref_count(struct ofgroup *group)
     OVS_EXCLUDED(ofproto_mutex)
 {
-    struct ofproto *ofproto = group->ofproto;
+    struct ofproto *ofproto = CONST_CAST(struct ofproto *, group->ofproto);
     struct rule_criteria criteria;
     struct rule_collection rules;
     struct match match;
@@ -5415,10 +5443,9 @@ group_get_ref_count(struct ofgroup *group)
 
 static void
 append_group_stats(struct ofgroup *group, struct list *replies)
-    OVS_REQ_RDLOCK(group->rwlock)
 {
     struct ofputil_group_stats ogs;
-    struct ofproto *ofproto = group->ofproto;
+    const struct ofproto *ofproto = group->ofproto;
     long long int now = time_msec();
     int error;
 
@@ -5446,64 +5473,64 @@ append_group_stats(struct ofgroup *group, struct list *replies)
     free(ogs.bucket_stats);
 }
 
-static enum ofperr
-handle_group_stats_request(struct ofconn *ofconn,
-                           const struct ofp_header *request)
+static void
+handle_group_request(struct ofconn *ofconn,
+                     const struct ofp_header *request, uint32_t group_id,
+                     void (*cb)(struct ofgroup *, struct list *replies))
 {
     struct ofproto *ofproto = ofconn_get_ofproto(ofconn);
-    struct list replies;
-    enum ofperr error;
     struct ofgroup *group;
-    uint32_t group_id;
-
-    error = ofputil_decode_group_stats_request(request, &group_id);
-    if (error) {
-        return error;
-    }
+    struct list replies;
 
     ofpmp_init(&replies, request);
-
     if (group_id == OFPG_ALL) {
         ovs_rwlock_rdlock(&ofproto->groups_rwlock);
         HMAP_FOR_EACH (group, hmap_node, &ofproto->groups) {
-            ovs_rwlock_rdlock(&group->rwlock);
-            append_group_stats(group, &replies);
-            ovs_rwlock_unlock(&group->rwlock);
+            cb(group, &replies);
         }
         ovs_rwlock_unlock(&ofproto->groups_rwlock);
     } else {
         if (ofproto_group_lookup(ofproto, group_id, &group)) {
-            append_group_stats(group, &replies);
-            ofproto_group_release(group);
+            cb(group, &replies);
+            ofproto_group_unref(group);
         }
     }
-
     ofconn_send_replies(ofconn, &replies);
-
-    return 0;
 }
 
 static enum ofperr
-handle_group_desc_stats_request(struct ofconn *ofconn,
-                                const struct ofp_header *request)
+handle_group_stats_request(struct ofconn *ofconn,
+                           const struct ofp_header *request)
 {
-    struct ofproto *ofproto = ofconn_get_ofproto(ofconn);
-    struct list replies;
-    struct ofputil_group_desc gds;
-    struct ofgroup *group;
-
-    ofpmp_init(&replies, request);
+    uint32_t group_id;
+    enum ofperr error;
 
-    ovs_rwlock_rdlock(&ofproto->groups_rwlock);
-    HMAP_FOR_EACH (group, hmap_node, &ofproto->groups) {
-        gds.group_id = group->group_id;
-        gds.type = group->type;
-        ofputil_append_group_desc_reply(&gds, &group->buckets, &replies);
+    error = ofputil_decode_group_stats_request(request, &group_id);
+    if (error) {
+        return error;
     }
-    ovs_rwlock_unlock(&ofproto->groups_rwlock);
 
-    ofconn_send_replies(ofconn, &replies);
+    handle_group_request(ofconn, request, group_id, append_group_stats);
+    return 0;
+}
+
+static void
+append_group_desc(struct ofgroup *group, struct list *replies)
+{
+    struct ofputil_group_desc gds;
 
+    gds.group_id = group->group_id;
+    gds.type = group->type;
+    ofputil_append_group_desc_reply(&gds, &group->buckets, replies);
+}
+
+static enum ofperr
+handle_group_desc_stats_request(struct ofconn *ofconn,
+                                const struct ofp_header *request)
+{
+    handle_group_request(ofconn, request,
+                         ofputil_decode_group_desc_request(request),
+                         append_group_desc);
     return 0;
 }
 
@@ -5565,6 +5592,46 @@ handle_queue_get_config_request(struct ofconn *ofconn,
    return 0;
 }
 
+static enum ofperr
+init_group(struct ofproto *ofproto, struct ofputil_group_mod *gm,
+           struct ofgroup **ofgroup)
+{
+    enum ofperr error;
+    const long long int now = time_msec();
+
+    if (gm->group_id > OFPG_MAX) {
+        return OFPERR_OFPGMFC_INVALID_GROUP;
+    }
+    if (gm->type > OFPGT11_FF) {
+        return OFPERR_OFPGMFC_BAD_TYPE;
+    }
+
+    *ofgroup = ofproto->ofproto_class->group_alloc();
+    if (!*ofgroup) {
+        VLOG_WARN_RL(&rl, "%s: failed to allocate group", ofproto->name);
+        return OFPERR_OFPGMFC_OUT_OF_GROUPS;
+    }
+
+    (*ofgroup)->ofproto = ofproto;
+    *CONST_CAST(uint32_t *, &((*ofgroup)->group_id)) = gm->group_id;
+    *CONST_CAST(enum ofp11_group_type *, &(*ofgroup)->type) = gm->type;
+    *CONST_CAST(long long int *, &((*ofgroup)->created)) = now;
+    *CONST_CAST(long long int *, &((*ofgroup)->modified)) = now;
+    ovs_refcount_init(&(*ofgroup)->ref_count);
+
+    list_move(&(*ofgroup)->buckets, &gm->buckets);
+    *CONST_CAST(uint32_t *, &(*ofgroup)->n_buckets) =
+        list_size(&(*ofgroup)->buckets);
+
+    /* Construct called BEFORE any locks are held. */
+    error = ofproto->ofproto_class->group_construct(*ofgroup);
+    if (error) {
+        ofputil_bucket_list_destroy(&(*ofgroup)->buckets);
+        ofproto->ofproto_class->group_dealloc(*ofgroup);
+    }
+    return error;
+}
+
 /* Implements OFPGC11_ADD
  * in which no matching flow already exists in the flow table.
  *
@@ -5584,33 +5651,10 @@ add_group(struct ofproto *ofproto, struct ofputil_group_mod *gm)
     struct ofgroup *ofgroup;
     enum ofperr error;
 
-    if (gm->group_id > OFPG_MAX) {
-        return OFPERR_OFPGMFC_INVALID_GROUP;
-    }
-    if (gm->type > OFPGT11_FF) {
-        return OFPERR_OFPGMFC_BAD_TYPE;
-    }
-
     /* Allocate new group and initialize it. */
-    ofgroup = ofproto->ofproto_class->group_alloc();
-    if (!ofgroup) {
-        VLOG_WARN_RL(&rl, "%s: failed to create group", ofproto->name);
-        return OFPERR_OFPGMFC_OUT_OF_GROUPS;
-    }
-
-    ovs_rwlock_init(&ofgroup->rwlock);
-    ofgroup->ofproto  = ofproto;
-    ofgroup->group_id = gm->group_id;
-    ofgroup->type     = gm->type;
-    ofgroup->created = ofgroup->modified = time_msec();
-
-    list_move(&ofgroup->buckets, &gm->buckets);
-    ofgroup->n_buckets = list_size(&ofgroup->buckets);
-
-    /* Construct called BEFORE any locks are held. */
-    error = ofproto->ofproto_class->group_construct(ofgroup);
+    error = init_group(ofproto, gm, &ofgroup);
     if (error) {
-        goto free_out;
+        return error;
     }
 
     /* We wrlock as late as possible to minimize the time we jam any other
@@ -5640,7 +5684,6 @@ add_group(struct ofproto *ofproto, struct ofputil_group_mod *gm)
  unlock_out:
     ovs_rwlock_unlock(&ofproto->groups_rwlock);
     ofproto->ofproto_class->group_destruct(ofgroup);
- free_out:
     ofputil_bucket_list_destroy(&ofgroup->buckets);
     ofproto->ofproto_class->group_dealloc(ofgroup);
 
@@ -5650,66 +5693,59 @@ add_group(struct ofproto *ofproto, struct ofputil_group_mod *gm)
 /* Implements OFPFC_MODIFY.  Returns 0 on success or an OpenFlow error code on
  * failure.
  *
+ * Note that the group is re-created and then replaces the old group in
+ * ofproto's ofgroup hash map. Thus, the group is never altered while users of
+ * the xlate module hold a pointer to the group.
+ *
  * 'ofconn' is used to retrieve the packet buffer specified in fm->buffer_id,
  * if any. */
 static enum ofperr
 modify_group(struct ofproto *ofproto, struct ofputil_group_mod *gm)
 {
-    struct ofgroup *ofgroup;
-    struct ofgroup *victim;
+    struct ofgroup *ofgroup, *new_ofgroup, *retiring;
     enum ofperr error;
 
-    if (gm->group_id > OFPG_MAX) {
-        return OFPERR_OFPGMFC_INVALID_GROUP;
-    }
-
-    if (gm->type > OFPGT11_FF) {
-        return OFPERR_OFPGMFC_BAD_TYPE;
+    error = init_group(ofproto, gm, &new_ofgroup);
+    if (error) {
+        return error;
     }
 
-    victim = ofproto->ofproto_class->group_alloc();
-    if (!victim) {
-        VLOG_WARN_RL(&rl, "%s: failed to allocate group", ofproto->name);
-        return OFPERR_OFPGMFC_OUT_OF_GROUPS;
-    }
+    retiring = new_ofgroup;
 
     if (!ofproto_group_write_lookup(ofproto, gm->group_id, &ofgroup)) {
         error = OFPERR_OFPGMFC_UNKNOWN_GROUP;
-        goto free_out;
+        goto out;
     }
-    /* Both group's and its container's write locks held now.
-     * Also, n_groups[] is protected by ofproto->groups_rwlock. */
+
+    /* Ofproto's group write lock is held now. */
     if (ofgroup->type != gm->type
         && ofproto->n_groups[gm->type] >= ofproto->ogf.max_groups[gm->type]) {
         error = OFPERR_OFPGMFC_OUT_OF_GROUPS;
-        goto unlock_out;
+        goto out;
     }
 
-    *victim = *ofgroup;
-    list_move(&victim->buckets, &ofgroup->buckets);
-
-    ofgroup->type = gm->type;
-    list_move(&ofgroup->buckets, &gm->buckets);
-    ofgroup->n_buckets = list_size(&ofgroup->buckets);
+    /* The group creation time does not change during modification. */
+    *CONST_CAST(long long int *, &(new_ofgroup->created)) = ofgroup->created;
+    *CONST_CAST(long long int *, &(new_ofgroup->modified)) = time_msec();
 
-    error = ofproto->ofproto_class->group_modify(ofgroup, victim);
-    if (!error) {
-        ofputil_bucket_list_destroy(&victim->buckets);
-        ofproto->n_groups[victim->type]--;
-        ofproto->n_groups[ofgroup->type]++;
-        ofgroup->modified = time_msec();
-    } else {
-        ofputil_bucket_list_destroy(&ofgroup->buckets);
+    error = ofproto->ofproto_class->group_modify(new_ofgroup);
+    if (error) {
+        goto out;
+    }
 
-        *ofgroup = *victim;
-        list_move(&ofgroup->buckets, &victim->buckets);
+    retiring = ofgroup;
+    /* Replace ofgroup in ofproto's groups hash map with new_ofgroup. */
+    hmap_remove(&ofproto->groups, &ofgroup->hmap_node);
+    hmap_insert(&ofproto->groups, &new_ofgroup->hmap_node,
+                hash_int(new_ofgroup->group_id, 0));
+    if (ofgroup->type != new_ofgroup->type) {
+        ofproto->n_groups[ofgroup->type]--;
+        ofproto->n_groups[new_ofgroup->type]++;
     }
 
- unlock_out:
-    ovs_rwlock_unlock(&ofgroup->rwlock);
+out:
+    ofproto_group_unref(retiring);
     ovs_rwlock_unlock(&ofproto->groups_rwlock);
- free_out:
-    ofproto->ofproto_class->group_dealloc(victim);
     return error;
 }
 
@@ -5726,19 +5762,11 @@ delete_group__(struct ofproto *ofproto, struct ofgroup *ofgroup)
     fm.out_group = ofgroup->group_id;
     handle_flow_mod__(ofproto, NULL, &fm, NULL);
 
-    /* Must wait until existing readers are done,
-     * while holding the container's write lock at the same time. */
-    ovs_rwlock_wrlock(&ofgroup->rwlock);
     hmap_remove(&ofproto->groups, &ofgroup->hmap_node);
     /* No-one can find this group any more. */
     ofproto->n_groups[ofgroup->type]--;
     ovs_rwlock_unlock(&ofproto->groups_rwlock);
-
-    ofproto->ofproto_class->group_destruct(ofgroup);
-    ofputil_bucket_list_destroy(&ofgroup->buckets);
-    ovs_rwlock_unlock(&ofgroup->rwlock);
-    ovs_rwlock_destroy(&ofgroup->rwlock);
-    ofproto->ofproto_class->group_dealloc(ofgroup);
+    ofproto_group_unref(ofgroup);
 }
 
 /* Implements OFPGC_DELETE. */
@@ -5861,6 +5889,69 @@ handle_table_mod(struct ofconn *ofconn, const struct ofp_header *oh)
     return table_mod(ofproto, &tm);
 }
 
+static enum ofperr
+handle_bundle_control(struct ofconn *ofconn, const struct ofp_header *oh)
+{
+    enum ofperr error;
+    struct ofputil_bundle_ctrl_msg bctrl;
+    struct ofpbuf *buf;
+    struct ofputil_bundle_ctrl_msg reply;
+
+    error = ofputil_decode_bundle_ctrl(oh, &bctrl);
+    if (error) {
+        return error;
+    }
+    reply.flags = 0;
+    reply.bundle_id = bctrl.bundle_id;
+
+    switch (bctrl.type) {
+        case OFPBCT_OPEN_REQUEST:
+        error = ofp_bundle_open(ofconn, bctrl.bundle_id, bctrl.flags);
+        reply.type = OFPBCT_OPEN_REPLY;
+        break;
+    case OFPBCT_CLOSE_REQUEST:
+        error = ofp_bundle_close(ofconn, bctrl.bundle_id, bctrl.flags);
+        reply.type = OFPBCT_CLOSE_REPLY;;
+        break;
+    case OFPBCT_COMMIT_REQUEST:
+        error = ofp_bundle_commit(ofconn, bctrl.bundle_id, bctrl.flags);
+        reply.type = OFPBCT_COMMIT_REPLY;
+        break;
+    case OFPBCT_DISCARD_REQUEST:
+        error = ofp_bundle_discard(ofconn, bctrl.bundle_id);
+        reply.type = OFPBCT_DISCARD_REPLY;
+        break;
+
+    case OFPBCT_OPEN_REPLY:
+    case OFPBCT_CLOSE_REPLY:
+    case OFPBCT_COMMIT_REPLY:
+    case OFPBCT_DISCARD_REPLY:
+        return OFPERR_OFPBFC_BAD_TYPE;
+        break;
+    }
+
+    if (!error) {
+        buf = ofputil_encode_bundle_ctrl_reply(oh, &reply);
+        ofconn_send_reply(ofconn, buf);
+    }
+    return error;
+}
+
+
+static enum ofperr
+handle_bundle_add(struct ofconn *ofconn, const struct ofp_header *oh)
+{
+    enum ofperr error;
+    struct ofputil_bundle_add_msg badd;
+
+    error = ofputil_decode_bundle_add(oh, &badd);
+    if (error) {
+        return error;
+    }
+
+    return ofp_bundle_add_message(ofconn, &badd);
+}
+
 static enum ofperr
 handle_openflow__(struct ofconn *ofconn, const struct ofpbuf *msg)
     OVS_EXCLUDED(ofproto_mutex)
@@ -5993,6 +6084,12 @@ handle_openflow__(struct ofconn *ofconn, const struct ofpbuf *msg)
     case OFPTYPE_QUEUE_GET_CONFIG_REQUEST:
         return handle_queue_get_config_request(ofconn, oh);
 
+    case OFPTYPE_BUNDLE_CONTROL:
+        return handle_bundle_control(ofconn, oh);
+
+    case OFPTYPE_BUNDLE_ADD_MESSAGE:
+        return handle_bundle_add(ofconn, oh);
+
     case OFPTYPE_HELLO:
     case OFPTYPE_ERROR:
     case OFPTYPE_FEATURES_REPLY:
@@ -6264,7 +6361,7 @@ ofopgroup_complete(struct ofopgroup *group)
                 rule->hard_timeout = op->hard_timeout;
                 ovs_mutex_unlock(&rule->mutex);
                 if (op->actions) {
-                    struct rule_actions *old_actions;
+                    const struct rule_actions *old_actions;
 
                     ovs_mutex_lock(&rule->mutex);
                     old_actions = rule_get_actions(rule);
@@ -6702,6 +6799,14 @@ oftable_init(struct oftable *table)
     classifier_init(&table->cls, flow_segment_u32s);
     table->max_flows = UINT_MAX;
     atomic_init(&table->config, (unsigned int)OFPROTO_TABLE_MISS_DEFAULT);
+
+    fat_rwlock_wrlock(&table->cls.rwlock);
+    classifier_set_prefix_fields(&table->cls, default_prefix_fields,
+                                 ARRAY_SIZE(default_prefix_fields));
+    fat_rwlock_unlock(&table->cls.rwlock);
+
+    atomic_init(&table->n_matched, 0);
+    atomic_init(&table->n_missed, 0);
 }
 
 /* Destroys 'table', including its classifier and eviction groups.
@@ -6840,7 +6945,7 @@ oftable_insert_rule(struct rule *rule)
 {
     struct ofproto *ofproto = rule->ofproto;
     struct oftable *table = &ofproto->tables[rule->table_id];
-    struct rule_actions *actions;
+    const struct rule_actions *actions;
     bool may_expire;
 
     ovs_mutex_lock(&rule->mutex);
@@ -6922,25 +7027,30 @@ ofproto_unixctl_init(void)
 void
 ofproto_get_vlan_usage(struct ofproto *ofproto, unsigned long int *vlan_bitmap)
 {
+    struct match match;
+    struct cls_rule target;
     const struct oftable *oftable;
 
+    match_init_catchall(&match);
+    match_set_vlan_vid_masked(&match, htons(VLAN_CFI), htons(VLAN_CFI));
+    cls_rule_init(&target, &match, 0);
+
     free(ofproto->vlan_bitmap);
     ofproto->vlan_bitmap = bitmap_allocate(4096);
     ofproto->vlans_changed = false;
 
     OFPROTO_FOR_EACH_TABLE (oftable, ofproto) {
-        const struct cls_subtable *table;
+        struct cls_cursor cursor;
+        struct rule *rule;
 
         fat_rwlock_rdlock(&oftable->cls.rwlock);
-        HMAP_FOR_EACH (table, hmap_node, &oftable->cls.subtables) {
-            if (minimask_get_vid_mask(&table->mask) == VLAN_VID_MASK) {
-                const struct cls_rule *rule;
-
-                HMAP_FOR_EACH (rule, hmap_node, &table->rules) {
-                    uint16_t vid = miniflow_get_vid(&rule->match.flow);
-                    bitmap_set1(vlan_bitmap, vid);
-                    bitmap_set1(ofproto->vlan_bitmap, vid);
-                }
+        cls_cursor_init(&cursor, &oftable->cls, &target);
+        CLS_CURSOR_FOR_EACH (rule, cr, &cursor) {
+            if (minimask_get_vid_mask(&rule->cr.match.mask) == VLAN_VID_MASK) {
+                uint16_t vid = miniflow_get_vid(&rule->cr.match.flow);
+
+                bitmap_set1(vlan_bitmap, vid);
+                bitmap_set1(ofproto->vlan_bitmap, vid);
             }
         }
         fat_rwlock_unlock(&oftable->cls.rwlock);