ofproto: Mark immutable members of struct rule 'const'.
[cascardo/ovs.git] / ofproto / ofproto.c
index 5473c7d..58042c2 100644 (file)
@@ -153,10 +153,10 @@ static void oftable_enable_eviction(struct oftable *,
                                     const struct mf_subfield *fields,
                                     size_t n_fields);
 
-static void oftable_remove_rule(struct rule *rule) OVS_RELEASES(rule->rwlock);
+static void oftable_remove_rule(struct rule *rule) OVS_RELEASES(rule->mutex);
 static void oftable_remove_rule__(struct ofproto *ofproto,
                                   struct classifier *cls, struct rule *rule)
-    OVS_REQ_WRLOCK(cls->rwlock) OVS_RELEASES(rule->rwlock);
+    OVS_REQ_WRLOCK(cls->rwlock) OVS_RELEASES(rule->mutex);
 static void oftable_insert_rule(struct rule *);
 
 /* A set of rules within a single OpenFlow table (oftable) that have the same
@@ -182,7 +182,7 @@ struct eviction_group {
 };
 
 static bool choose_rule_to_evict(struct oftable *table, struct rule **rulep)
-    OVS_TRY_WRLOCK(true, (*rulep)->rwlock);
+    OVS_TRY_WRLOCK(true, (*rulep)->mutex);
 static void ofproto_evict(struct ofproto *);
 static uint32_t rule_eviction_priority(struct rule *);
 static void eviction_group_add_rule(struct rule *);
@@ -218,6 +218,17 @@ static void rule_criteria_init(struct rule_criteria *, uint8_t table_id,
                                ofp_port_t out_port);
 static void rule_criteria_destroy(struct rule_criteria *);
 
+/* A packet that needs to be passed to rule_execute(). */
+struct rule_execute {
+    struct list list_node;      /* In struct ofproto's "rule_executes" list. */
+    struct rule *rule;          /* Owns a reference to the rule. */
+    ofp_port_t in_port;
+    struct ofpbuf *packet;      /* Owns the packet. */
+};
+
+static void run_rule_executes(struct ofproto *);
+static void destroy_rule_executes(struct ofproto *);
+
 /* ofport. */
 static void ofport_destroy__(struct ofport *);
 static void ofport_destroy(struct ofport *);
@@ -241,7 +252,7 @@ static enum ofperr modify_flows__(struct ofproto *, struct ofconn *,
                                   const struct rule_collection *);
 static void delete_flow__(struct rule *rule, struct ofopgroup *,
                           enum ofp_flow_removed_reason)
-    OVS_RELEASES(rule->rwlock);
+    OVS_RELEASES(rule->mutex);
 static bool handle_openflow(struct ofconn *, const struct ofpbuf *);
 static enum ofperr handle_flow_mod__(struct ofproto *, struct ofconn *,
                                      struct ofputil_flow_mod *,
@@ -264,6 +275,8 @@ static const struct ofproto_class **ofproto_classes;
 static size_t n_ofproto_classes;
 static size_t allocated_ofproto_classes;
 
+struct ovs_mutex ofproto_mutex;
+
 unsigned flow_eviction_threshold = OFPROTO_FLOW_EVICTION_THRESHOLD_DEFAULT;
 unsigned n_handler_threads;
 enum ofproto_flow_miss_model flow_miss_model = OFPROTO_HANDLE_MISS_AUTO;
@@ -294,6 +307,8 @@ ofproto_init(const struct shash *iface_hints)
     struct shash_node *node;
     size_t i;
 
+    ovs_mutex_init_recursive(&ofproto_mutex);
+
     ofproto_class_register(&ofproto_dpif_class);
 
     /* Make a local copy, since we don't own 'iface_hints' elements. */
@@ -449,6 +464,7 @@ ofproto_create(const char *datapath_name, const char *datapath_type,
     }
 
     /* Initialize. */
+    ovs_mutex_lock(&ofproto_mutex);
     memset(ofproto, 0, sizeof *ofproto);
     ofproto->ofproto_class = class;
     ofproto->name = xstrdup(datapath_name);
@@ -473,12 +489,12 @@ ofproto_create(const char *datapath_name, const char *datapath_type,
     ofproto->n_tables = 0;
     hindex_init(&ofproto->cookies);
     list_init(&ofproto->expirable);
-    ovs_mutex_init_recursive(&ofproto->expirable_mutex);
     ofproto->connmgr = connmgr_create(ofproto, datapath_name, datapath_name);
     ofproto->state = S_OPENFLOW;
     list_init(&ofproto->pending);
     ofproto->n_pending = 0;
     hmap_init(&ofproto->deletions);
+    guarded_list_init(&ofproto->rule_executes);
     ofproto->n_add = ofproto->n_delete = ofproto->n_modify = 0;
     ofproto->first_op = ofproto->last_op = LLONG_MIN;
     ofproto->next_op_report = LLONG_MAX;
@@ -486,6 +502,7 @@ ofproto_create(const char *datapath_name, const char *datapath_type,
     ofproto->vlan_bitmap = NULL;
     ofproto->vlans_changed = false;
     ofproto->min_mtu = INT_MAX;
+    ovs_mutex_unlock(&ofproto_mutex);
 
     error = ofproto->ofproto_class->construct(ofproto);
     if (error) {
@@ -1131,7 +1148,7 @@ ofproto_rule_delete(struct ofproto *ofproto, struct classifier *cls,
 
     group = ofopgroup_create_unattached(ofproto);
     ofoperation_create(group, rule, OFOPERATION_DELETE, OFPRR_DELETE);
-    ovs_rwlock_wrlock(&rule->rwlock);
+    ovs_mutex_lock(&rule->mutex);
     oftable_remove_rule__(ofproto, cls, rule);
     ofproto->ofproto_class->rule_delete(rule);
     ofopgroup_submit(group);
@@ -1173,6 +1190,9 @@ ofproto_destroy__(struct ofproto *ofproto)
     ovs_assert(list_is_empty(&ofproto->pending));
     ovs_assert(!ofproto->n_pending);
 
+    destroy_rule_executes(ofproto);
+    guarded_list_destroy(&ofproto->rule_executes);
+
     connmgr_destroy(ofproto->connmgr);
 
     hmap_remove(&all_ofprotos, &ofproto->hmap_node);
@@ -1197,7 +1217,6 @@ ofproto_destroy__(struct ofproto *ofproto)
 
     free(ofproto->vlan_bitmap);
 
-    ovs_mutex_destroy(&ofproto->expirable_mutex);
     ofproto->ofproto_class->dealloc(ofproto);
 }
 
@@ -1312,6 +1331,8 @@ ofproto_run(struct ofproto *p)
         VLOG_ERR_RL(&rl, "%s: run failed (%s)", p->name, ovs_strerror(error));
     }
 
+    run_rule_executes(p);
+
     /* Restore the eviction group heap invariant occasionally. */
     if (p->eviction_group_timer < time_msec()) {
         size_t i;
@@ -1769,11 +1790,11 @@ ofproto_add_flow(struct ofproto *ofproto, const struct match *match,
     rule = rule_from_cls_rule(classifier_find_match_exactly(
                                   &ofproto->tables[0].cls, match, priority));
     if (rule) {
-        ovs_rwlock_rdlock(&rule->rwlock);
+        ovs_mutex_lock(&rule->mutex);
         must_add = !ofpacts_equal(rule->actions->ofpacts,
                                   rule->actions->ofpacts_len,
                                   ofpacts, ofpacts_len);
-        ovs_rwlock_unlock(&rule->rwlock);
+        ovs_mutex_unlock(&rule->mutex);
     } else {
         must_add = true;
     }
@@ -2346,10 +2367,9 @@ ofproto_rule_unref(struct rule *rule)
 static void
 ofproto_rule_destroy__(struct rule *rule)
 {
-    cls_rule_destroy(&rule->cr);
+    cls_rule_destroy(CONST_CAST(struct cls_rule *, &rule->cr));
     rule_actions_unref(rule->actions);
-    ovs_mutex_destroy(&rule->timeout_mutex);
-    ovs_rwlock_destroy(&rule->rwlock);
+    ovs_mutex_destroy(&rule->mutex);
     rule->ofproto->ofproto_class->rule_dealloc(rule);
 }
 
@@ -2399,7 +2419,7 @@ rule_actions_unref(struct rule_actions *actions)
 
 /* Returns true if 'rule' has an OpenFlow OFPAT_OUTPUT or OFPAT_ENQUEUE action
  * that outputs to 'port' (output to OFPP_FLOOD and OFPP_ALL doesn't count). */
-bool
+static bool
 ofproto_rule_has_out_port(const struct rule *rule, ofp_port_t port)
 {
     return (port == OFPP_ANY
@@ -2430,22 +2450,48 @@ ofoperation_has_out_port(const struct ofoperation *op, ofp_port_t out_port)
     NOT_REACHED();
 }
 
-/* Executes the actions indicated by 'rule' on 'packet' and credits 'rule''s
- * statistics appropriately.
- *
- * 'packet' doesn't necessarily have to match 'rule'.  'rule' will be credited
- * with statistics for 'packet' either way.
- *
- * Takes ownership of 'packet'. */
-static int
-rule_execute(struct rule *rule, ofp_port_t in_port, struct ofpbuf *packet)
+static void
+rule_execute_destroy(struct rule_execute *e)
 {
-    struct flow flow;
-    union flow_in_port in_port_;
+    ofproto_rule_unref(e->rule);
+    list_remove(&e->list_node);
+    free(e);
+}
+
+/* Executes all "rule_execute" operations queued up in ofproto->rule_executes,
+ * by passing them to the ofproto provider. */
+static void
+run_rule_executes(struct ofproto *ofproto)
+{
+    struct rule_execute *e, *next;
+    struct list executes;
 
-    in_port_.ofp_port = in_port;
-    flow_extract(packet, 0, 0, NULL, &in_port_, &flow);
-    return rule->ofproto->ofproto_class->rule_execute(rule, &flow, packet);
+    guarded_list_pop_all(&ofproto->rule_executes, &executes);
+    LIST_FOR_EACH_SAFE (e, next, list_node, &executes) {
+        union flow_in_port in_port_;
+        struct flow flow;
+
+        in_port_.ofp_port = e->in_port;
+        flow_extract(e->packet, 0, 0, NULL, &in_port_, &flow);
+        ofproto->ofproto_class->rule_execute(e->rule, &flow, e->packet);
+
+        rule_execute_destroy(e);
+    }
+}
+
+/* Destroys and discards all "rule_execute" operations queued up in
+ * ofproto->rule_executes. */
+static void
+destroy_rule_executes(struct ofproto *ofproto)
+{
+    struct rule_execute *e, *next;
+    struct list executes;
+
+    guarded_list_pop_all(&ofproto->rule_executes, &executes);
+    LIST_FOR_EACH_SAFE (e, next, list_node, &executes) {
+        ofpbuf_delete(e->packet);
+        rule_execute_destroy(e);
+    }
 }
 
 /* Returns true if 'rule' should be hidden from the controller.
@@ -2453,7 +2499,7 @@ rule_execute(struct rule *rule, ofp_port_t in_port, struct ofpbuf *packet)
  * Rules with priority higher than UINT16_MAX are set up by ofproto itself
  * (e.g. by in-band control) and are intentionally hidden from the
  * controller. */
-bool
+static bool
 ofproto_rule_is_hidden(const struct rule *rule)
 {
     return rule->cr.priority > UINT16_MAX;
@@ -2909,6 +2955,7 @@ hash_cookie(ovs_be64 cookie)
 
 static void
 cookies_insert(struct ofproto *ofproto, struct rule *rule)
+    OVS_REQUIRES(ofproto_mutex)
 {
     hindex_insert(&ofproto->cookies, &rule->cookie_node,
                   hash_cookie(rule->flow_cookie));
@@ -2916,6 +2963,7 @@ cookies_insert(struct ofproto *ofproto, struct rule *rule)
 
 static void
 cookies_remove(struct ofproto *ofproto, struct rule *rule)
+    OVS_REQUIRES(ofproto_mutex)
 {
     hindex_remove(&ofproto->cookies, &rule->cookie_node);
 }
@@ -2925,13 +2973,15 @@ ofproto_rule_change_cookie(struct ofproto *ofproto, struct rule *rule,
                            ovs_be64 new_cookie)
 {
     if (new_cookie != rule->flow_cookie) {
+        ovs_mutex_lock(&ofproto_mutex);
         cookies_remove(ofproto, rule);
 
-        ovs_rwlock_wrlock(&rule->rwlock);
+        ovs_mutex_lock(&rule->mutex);
         rule->flow_cookie = new_cookie;
-        ovs_rwlock_unlock(&rule->rwlock);
+        ovs_mutex_unlock(&rule->mutex);
 
         cookies_insert(ofproto, rule);
+        ovs_mutex_unlock(&ofproto_mutex);
     }
 }
 
@@ -3117,6 +3167,7 @@ collect_rules_loose(struct ofproto *ofproto,
     if (criteria->cookie_mask == htonll(UINT64_MAX)) {
         struct rule *rule;
 
+        ovs_mutex_lock(&ofproto_mutex);
         HINDEX_FOR_EACH_WITH_HASH (rule, cookie_node,
                                    hash_cookie(criteria->cookie),
                                    &ofproto->cookies) {
@@ -3127,6 +3178,7 @@ collect_rules_loose(struct ofproto *ofproto,
                 }
             }
         }
+        ovs_mutex_unlock(&ofproto_mutex);
     } else {
         FOR_EACH_MATCHING_TABLE (table, criteria->table_id, ofproto) {
             struct cls_cursor cursor;
@@ -3177,6 +3229,7 @@ collect_rules_strict(struct ofproto *ofproto,
     if (criteria->cookie_mask == htonll(UINT64_MAX)) {
         struct rule *rule;
 
+        ovs_mutex_lock(&ofproto_mutex);
         HINDEX_FOR_EACH_WITH_HASH (rule, cookie_node,
                                    hash_cookie(criteria->cookie),
                                    &ofproto->cookies) {
@@ -3187,6 +3240,7 @@ collect_rules_strict(struct ofproto *ofproto,
                 }
             }
         }
+        ovs_mutex_unlock(&ofproto_mutex);
     } else {
         FOR_EACH_MATCHING_TABLE (table, criteria->table_id, ofproto) {
             struct rule *rule;
@@ -3264,10 +3318,10 @@ handle_flow_stats_request(struct ofconn *ofconn,
         fs.ofpacts = rule->actions->ofpacts;
         fs.ofpacts_len = rule->actions->ofpacts_len;
 
-        ovs_mutex_lock(&rule->timeout_mutex);
+        ovs_mutex_lock(&rule->mutex);
         fs.idle_timeout = rule->idle_timeout;
         fs.hard_timeout = rule->hard_timeout;
-        ovs_mutex_unlock(&rule->timeout_mutex);
+        ovs_mutex_unlock(&rule->mutex);
 
         fs.flags = 0;
         if (rule->send_flow_removed) {
@@ -3552,7 +3606,7 @@ evict_rule_from_table(struct ofproto *ofproto, struct oftable *table)
     } else if (!choose_rule_to_evict(table, &rule)) {
         return OFPERR_OFPFMFC_TABLE_FULL;
     } else if (rule->pending) {
-        ovs_rwlock_unlock(&rule->rwlock);
+        ovs_mutex_unlock(&rule->mutex);
         return OFPROTO_POSTPONE;
     } else {
         struct ofopgroup *group;
@@ -3688,20 +3742,20 @@ add_flow(struct ofproto *ofproto, struct ofconn *ofconn,
     }
 
     /* Initialize base state. */
-    rule->ofproto = ofproto;
-    cls_rule_move(&rule->cr, &cr);
+    *CONST_CAST(struct ofproto **, &rule->ofproto) = ofproto;
+    cls_rule_move(CONST_CAST(struct cls_rule *, &rule->cr), &cr);
     atomic_init(&rule->ref_count, 1);
     rule->pending = NULL;
     rule->flow_cookie = fm->new_cookie;
     rule->created = rule->modified = rule->used = time_msec();
 
-    ovs_mutex_init(&rule->timeout_mutex);
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_init(&rule->mutex);
+    ovs_mutex_lock(&rule->mutex);
     rule->idle_timeout = fm->idle_timeout;
     rule->hard_timeout = fm->hard_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
 
-    rule->table_id = table - ofproto->tables;
+    *CONST_CAST(uint8_t *, &rule->table_id) = table - ofproto->tables;
     rule->send_flow_removed = (fm->flags & OFPUTIL_FF_SEND_FLOW_REM) != 0;
     rule->actions = rule_actions_create(fm->ofpacts, fm->ofpacts_len);
     list_init(&rule->meter_list_node);
@@ -3710,7 +3764,6 @@ add_flow(struct ofproto *ofproto, struct ofconn *ofconn,
     rule->monitor_flags = 0;
     rule->add_seqno = 0;
     rule->modify_seqno = 0;
-    ovs_rwlock_init(&rule->rwlock);
 
     /* Construct rule, initializing derived state. */
     error = ofproto->ofproto_class->rule_construct(rule);
@@ -3784,10 +3837,10 @@ modify_flows__(struct ofproto *ofproto, struct ofconn *ofconn,
             ofproto_rule_change_cookie(ofproto, rule, fm->new_cookie);
         }
         if (type == OFOPERATION_REPLACE) {
-            ovs_mutex_lock(&rule->timeout_mutex);
+            ovs_mutex_lock(&rule->mutex);
             rule->idle_timeout = fm->idle_timeout;
             rule->hard_timeout = fm->hard_timeout;
-            ovs_mutex_unlock(&rule->timeout_mutex);
+            ovs_mutex_unlock(&rule->mutex);
 
             rule->send_flow_removed = (fm->flags
                                        & OFPUTIL_FF_SEND_FLOW_REM) != 0;
@@ -3808,9 +3861,9 @@ modify_flows__(struct ofproto *ofproto, struct ofconn *ofconn,
             op->actions = rule->actions;
             new_actions = rule_actions_create(fm->ofpacts, fm->ofpacts_len);
 
-            ovs_rwlock_wrlock(&rule->rwlock);
+            ovs_mutex_lock(&rule->mutex);
             rule->actions = new_actions;
-            ovs_rwlock_unlock(&rule->rwlock);
+            ovs_mutex_unlock(&rule->mutex);
 
             rule->ofproto->ofproto_class->rule_modify_actions(rule,
                                                               reset_counters);
@@ -3925,7 +3978,7 @@ delete_flows__(struct ofproto *ofproto, struct ofconn *ofconn,
     group = ofopgroup_create(ofproto, ofconn, request, UINT32_MAX);
     for (i = 0; i < rules->n; i++) {
         struct rule *rule = rules->rules[i];
-        ovs_rwlock_wrlock(&rule->rwlock);
+        ovs_mutex_lock(&rule->mutex);
         delete_flow__(rule, group, reason);
     }
     ofopgroup_submit(group);
@@ -3996,10 +4049,10 @@ ofproto_rule_send_removed(struct rule *rule, uint8_t reason)
     fr.table_id = rule->table_id;
     calc_duration(rule->created, time_msec(),
                   &fr.duration_sec, &fr.duration_nsec);
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     fr.idle_timeout = rule->idle_timeout;
     fr.hard_timeout = rule->hard_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
     rule->ofproto->ofproto_class->rule_get_stats(rule, &fr.packet_count,
                                                  &fr.byte_count);
 
@@ -4047,22 +4100,22 @@ reduce_timeout(uint16_t max, uint16_t *timeout)
 void
 ofproto_rule_reduce_timeouts(struct rule *rule,
                              uint16_t idle_timeout, uint16_t hard_timeout)
-    OVS_EXCLUDED(rule->ofproto->expirable_mutex, rule->timeout_mutex)
+    OVS_EXCLUDED(ofproto_mutex, rule->mutex)
 {
     if (!idle_timeout && !hard_timeout) {
         return;
     }
 
-    ovs_mutex_lock(&rule->ofproto->expirable_mutex);
+    ovs_mutex_lock(&ofproto_mutex);
     if (list_is_empty(&rule->expirable)) {
         list_insert(&rule->ofproto->expirable, &rule->expirable);
     }
-    ovs_mutex_unlock(&rule->ofproto->expirable_mutex);
+    ovs_mutex_unlock(&ofproto_mutex);
 
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     reduce_timeout(idle_timeout, &rule->idle_timeout);
     reduce_timeout(hard_timeout, &rule->hard_timeout);
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
 }
 \f
 static enum ofperr
@@ -4126,35 +4179,46 @@ static enum ofperr
 handle_flow_mod__(struct ofproto *ofproto, struct ofconn *ofconn,
                   struct ofputil_flow_mod *fm, const struct ofp_header *oh)
 {
-    if (ofproto->n_pending >= 50) {
-        ovs_assert(!list_is_empty(&ofproto->pending));
-        return OFPROTO_POSTPONE;
-    }
+    enum ofperr error;
 
-    switch (fm->command) {
-    case OFPFC_ADD:
-        return add_flow(ofproto, ofconn, fm, oh);
+    if (ofproto->n_pending < 50) {
+        switch (fm->command) {
+        case OFPFC_ADD:
+            error = add_flow(ofproto, ofconn, fm, oh);
+            break;
 
-    case OFPFC_MODIFY:
-        return modify_flows_loose(ofproto, ofconn, fm, oh);
+        case OFPFC_MODIFY:
+            error = modify_flows_loose(ofproto, ofconn, fm, oh);
+            break;
 
-    case OFPFC_MODIFY_STRICT:
-        return modify_flow_strict(ofproto, ofconn, fm, oh);
+        case OFPFC_MODIFY_STRICT:
+            error = modify_flow_strict(ofproto, ofconn, fm, oh);
+            break;
 
-    case OFPFC_DELETE:
-        return delete_flows_loose(ofproto, ofconn, fm, oh);
+        case OFPFC_DELETE:
+            error = delete_flows_loose(ofproto, ofconn, fm, oh);
+            break;
 
-    case OFPFC_DELETE_STRICT:
-        return delete_flow_strict(ofproto, ofconn, fm, oh);
+        case OFPFC_DELETE_STRICT:
+            error = delete_flow_strict(ofproto, ofconn, fm, oh);
+            break;
 
-    default:
-        if (fm->command > 0xff) {
-            VLOG_WARN_RL(&rl, "%s: flow_mod has explicit table_id but "
-                         "flow_mod_table_id extension is not enabled",
-                         ofproto->name);
+        default:
+            if (fm->command > 0xff) {
+                VLOG_WARN_RL(&rl, "%s: flow_mod has explicit table_id but "
+                             "flow_mod_table_id extension is not enabled",
+                             ofproto->name);
+            }
+            error = OFPERR_OFPFMFC_BAD_COMMAND;
+            break;
         }
-        return OFPERR_OFPFMFC_BAD_COMMAND;
+    } else {
+        ovs_assert(!list_is_empty(&ofproto->pending));
+        error = OFPROTO_POSTPONE;
     }
+
+    run_rule_executes(ofproto);
+    return error;
 }
 
 static enum ofperr
@@ -4325,10 +4389,10 @@ ofproto_compose_flow_refresh_update(const struct rule *rule,
     fu.event = (flags & (NXFMF_INITIAL | NXFMF_ADD)
                 ? NXFME_ADDED : NXFME_MODIFIED);
     fu.reason = 0;
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     fu.idle_timeout = rule->idle_timeout;
     fu.hard_timeout = rule->hard_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
     fu.table_id = rule->table_id;
     fu.cookie = rule->flow_cookie;
     minimatch_expand(&rule->cr.match, &match);
@@ -5132,8 +5196,23 @@ ofopgroup_complete(struct ofopgroup *group)
                 error = ofconn_pktbuf_retrieve(group->ofconn, group->buffer_id,
                                                &packet, &in_port);
                 if (packet) {
+                    struct rule_execute *re;
+
                     ovs_assert(!error);
-                    error = rule_execute(op->rule, in_port, packet);
+
+                    ofproto_rule_ref(op->rule);
+
+                    re = xmalloc(sizeof *re);
+                    re->rule = op->rule;
+                    re->in_port = in_port;
+                    re->packet = packet;
+
+                    if (!guarded_list_push_back(&ofproto->rule_executes,
+                                                &re->list_node, 1024)) {
+                        ofproto_rule_unref(op->rule);
+                        ofpbuf_delete(re->packet);
+                        free(re);
+                    }
                 }
                 break;
             }
@@ -5209,7 +5288,7 @@ ofopgroup_complete(struct ofopgroup *group)
                     }
                 }
             } else {
-                ovs_rwlock_wrlock(&rule->rwlock);
+                ovs_mutex_lock(&rule->mutex);
                 oftable_remove_rule(rule);
                 ofproto_rule_unref(rule);
             }
@@ -5232,17 +5311,17 @@ ofopgroup_complete(struct ofopgroup *group)
                 }
             } else {
                 ofproto_rule_change_cookie(ofproto, rule, op->flow_cookie);
-                ovs_mutex_lock(&rule->timeout_mutex);
+                ovs_mutex_lock(&rule->mutex);
                 rule->idle_timeout = op->idle_timeout;
                 rule->hard_timeout = op->hard_timeout;
-                ovs_mutex_unlock(&rule->timeout_mutex);
+                ovs_mutex_unlock(&rule->mutex);
                 if (op->actions) {
                     struct rule_actions *old_actions;
 
-                    ovs_rwlock_wrlock(&rule->rwlock);
+                    ovs_mutex_lock(&rule->mutex);
                     old_actions = rule->actions;
                     rule->actions = op->actions;
-                    ovs_rwlock_unlock(&rule->rwlock);
+                    ovs_mutex_unlock(&rule->mutex);
 
                     op->actions = NULL;
                     rule_actions_unref(old_actions);
@@ -5301,10 +5380,10 @@ ofoperation_create(struct ofopgroup *group, struct rule *rule,
     op->type = type;
     op->reason = reason;
     op->flow_cookie = rule->flow_cookie;
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     op->idle_timeout = rule->idle_timeout;
     op->hard_timeout = rule->hard_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
     op->send_flow_removed = rule->send_flow_removed;
 
     group->n_running++;
@@ -5431,7 +5510,7 @@ choose_rule_to_evict(struct oftable *table, struct rule **rulep)
         struct rule *rule;
 
         HEAP_FOR_EACH (rule, evg_node, &evg->rules) {
-            if (!ovs_rwlock_trywrlock(&rule->rwlock)) {
+            if (!ovs_mutex_trylock(&rule->mutex)) {
                 *rulep = rule;
                 return true;
             }
@@ -5472,7 +5551,7 @@ ofproto_evict(struct ofproto *ofproto)
             }
 
             if (rule->pending) {
-                ovs_rwlock_unlock(&rule->rwlock);
+                ovs_mutex_unlock(&rule->mutex);
                 break;
             }
 
@@ -5616,7 +5695,7 @@ rule_eviction_priority(struct rule *rule)
     uint32_t expiration_offset;
 
     /* Calculate time of expiration. */
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     hard_expiration = (rule->hard_timeout
                        ? rule->modified + rule->hard_timeout * 1000
                        : LLONG_MAX);
@@ -5624,7 +5703,7 @@ rule_eviction_priority(struct rule *rule)
                        ? rule->used + rule->idle_timeout * 1000
                        : LLONG_MAX);
     expiration = MIN(hard_expiration, idle_expiration);
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
     if (expiration == LLONG_MAX) {
         return 0;
     }
@@ -5653,9 +5732,9 @@ eviction_group_add_rule(struct rule *rule)
     struct oftable *table = &ofproto->tables[rule->table_id];
     bool has_timeout;
 
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     has_timeout = rule->hard_timeout || rule->idle_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
 
     if (table->eviction_fields && has_timeout) {
         struct eviction_group *evg;
@@ -5780,21 +5859,25 @@ oftable_enable_eviction(struct oftable *table,
 static void
 oftable_remove_rule__(struct ofproto *ofproto, struct classifier *cls,
                       struct rule *rule)
-    OVS_REQ_WRLOCK(cls->rwlock) OVS_RELEASES(rule->rwlock)
+    OVS_REQ_WRLOCK(cls->rwlock) OVS_RELEASES(rule->mutex)
 {
-    classifier_remove(cls, &rule->cr);
+    classifier_remove(cls, CONST_CAST(struct cls_rule *, &rule->cr));
+
+    ovs_mutex_lock(&ofproto_mutex);
     cookies_remove(ofproto, rule);
+    ovs_mutex_unlock(&ofproto_mutex);
+
     eviction_group_remove_rule(rule);
-    ovs_mutex_lock(&ofproto->expirable_mutex);
+    ovs_mutex_lock(&ofproto_mutex);
     if (!list_is_empty(&rule->expirable)) {
         list_remove(&rule->expirable);
     }
-    ovs_mutex_unlock(&ofproto->expirable_mutex);
+    ovs_mutex_unlock(&ofproto_mutex);
     if (!list_is_empty(&rule->meter_list_node)) {
         list_remove(&rule->meter_list_node);
         list_init(&rule->meter_list_node);
     }
-    ovs_rwlock_unlock(&rule->rwlock);
+    ovs_mutex_unlock(&rule->mutex);
 }
 
 static void
@@ -5817,23 +5900,26 @@ oftable_insert_rule(struct rule *rule)
     struct oftable *table = &ofproto->tables[rule->table_id];
     bool may_expire;
 
-    ovs_mutex_lock(&rule->timeout_mutex);
+    ovs_mutex_lock(&rule->mutex);
     may_expire = rule->hard_timeout || rule->idle_timeout;
-    ovs_mutex_unlock(&rule->timeout_mutex);
+    ovs_mutex_unlock(&rule->mutex);
 
     if (may_expire) {
-        ovs_mutex_lock(&ofproto->expirable_mutex);
+        ovs_mutex_lock(&ofproto_mutex);
         list_insert(&ofproto->expirable, &rule->expirable);
-        ovs_mutex_unlock(&ofproto->expirable_mutex);
+        ovs_mutex_unlock(&ofproto_mutex);
     }
+
+    ovs_mutex_lock(&ofproto_mutex);
     cookies_insert(ofproto, rule);
+    ovs_mutex_unlock(&ofproto_mutex);
 
     if (rule->actions->meter_id) {
         struct meter *meter = ofproto->meters[rule->actions->meter_id];
         list_insert(&meter->rules, &rule->meter_list_node);
     }
     ovs_rwlock_wrlock(&table->cls.rwlock);
-    classifier_insert(&table->cls, &rule->cr);
+    classifier_insert(&table->cls, CONST_CAST(struct cls_rule *, &rule->cr));
     ovs_rwlock_unlock(&table->cls.rwlock);
     eviction_group_add_rule(rule);
 }