datapath: Pad OVS_PACKET_ATTR_PACKET if linear copy was performed
[cascardo/ovs.git] / datapath / datapath.c
index 940120c..9e9c73a 100644 (file)
@@ -61,8 +61,6 @@
 #include "vport-internal_dev.h"
 #include "vport-netdev.h"
 
-#define REHASH_FLOW_INTERVAL (10 * 60 * HZ)
-
 int ovs_net_id __read_mostly;
 
 static void ovs_notify(struct sk_buff *skb, struct genl_info *info,
@@ -112,10 +110,9 @@ int lockdep_ovsl_is_held(void)
 #endif
 
 static struct vport *new_vport(const struct vport_parms *);
-static int queue_gso_packets(struct net *, int dp_ifindex, struct sk_buff *,
+static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
                             const struct dp_upcall_info *);
-static int queue_userspace_packet(struct net *, int dp_ifindex,
-                                 struct sk_buff *,
+static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
                                  const struct dp_upcall_info *);
 
 /* Must be called with rcu_read_lock or ovs_mutex. */
@@ -165,7 +162,7 @@ static void destroy_dp_rcu(struct rcu_head *rcu)
 {
        struct datapath *dp = container_of(rcu, struct datapath, rcu);
 
-       ovs_flow_tbl_destroy((__force struct flow_table *)dp->table, false);
+       ovs_flow_tbl_destroy(&dp->table);
        free_percpu(dp->stats_percpu);
        release_net(ovs_dp_get_net(dp));
        kfree(dp->ports);
@@ -225,6 +222,7 @@ void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
        struct dp_stats_percpu *stats;
        struct sw_flow_key key;
        u64 *stats_counter;
+       u32 n_mask_hit;
        int error;
 
        stats = this_cpu_ptr(dp->stats_percpu);
@@ -237,7 +235,7 @@ void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
        }
 
        /* Look up flow. */
-       flow = ovs_flow_tbl_lookup(rcu_dereference(dp->table), &key);
+       flow = ovs_flow_tbl_lookup_stats(&dp->table, &key, &n_mask_hit);
        if (unlikely(!flow)) {
                struct dp_upcall_info upcall;
 
@@ -254,14 +252,15 @@ void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
        OVS_CB(skb)->flow = flow;
        OVS_CB(skb)->pkt_key = &key;
 
-       stats_counter = &stats->n_hit;
-       ovs_flow_used(OVS_CB(skb)->flow, skb);
+       ovs_flow_stats_update(OVS_CB(skb)->flow, skb);
        ovs_execute_actions(dp, skb);
+       stats_counter = &stats->n_hit;
 
 out:
        /* Update datapath statistics. */
        u64_stats_update_begin(&stats->sync);
        (*stats_counter)++;
+       stats->n_mask_hit += n_mask_hit;
        u64_stats_update_end(&stats->sync);
 }
 
@@ -279,7 +278,6 @@ int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
                  const struct dp_upcall_info *upcall_info)
 {
        struct dp_stats_percpu *stats;
-       int dp_ifindex;
        int err;
 
        if (upcall_info->portid == 0) {
@@ -287,16 +285,10 @@ int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
                goto err;
        }
 
-       dp_ifindex = get_dpifindex(dp);
-       if (!dp_ifindex) {
-               err = -ENODEV;
-               goto err;
-       }
-
        if (!skb_is_gso(skb))
-               err = queue_userspace_packet(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
+               err = queue_userspace_packet(dp, skb, upcall_info);
        else
-               err = queue_gso_packets(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
+               err = queue_gso_packets(dp, skb, upcall_info);
        if (err)
                goto err;
 
@@ -312,8 +304,7 @@ err:
        return err;
 }
 
-static int queue_gso_packets(struct net *net, int dp_ifindex,
-                            struct sk_buff *skb,
+static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
                             const struct dp_upcall_info *upcall_info)
 {
        unsigned short gso_type = skb_shinfo(skb)->gso_type;
@@ -322,14 +313,14 @@ static int queue_gso_packets(struct net *net, int dp_ifindex,
        struct sk_buff *segs, *nskb;
        int err;
 
-       segs = __skb_gso_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM, false);
+       segs = __skb_gso_segment(skb, NETIF_F_SG, false);
        if (IS_ERR(segs))
                return PTR_ERR(segs);
 
        /* Queue all of the segments. */
        skb = segs;
        do {
-               err = queue_userspace_packet(net, dp_ifindex, skb, upcall_info);
+               err = queue_userspace_packet(dp, skb, upcall_info);
                if (err)
                        break;
 
@@ -382,11 +373,11 @@ static size_t key_attr_size(void)
                + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 }
 
-static size_t upcall_msg_size(const struct sk_buff *skb,
-                             const struct nlattr *userdata)
+static size_t upcall_msg_size(const struct nlattr *userdata,
+                             unsigned int hdrlen)
 {
        size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
-               + nla_total_size(skb->len) /* OVS_PACKET_ATTR_PACKET */
+               + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
                + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 
        /* OVS_PACKET_ATTR_USERDATA */
@@ -396,15 +387,26 @@ static size_t upcall_msg_size(const struct sk_buff *skb,
        return size;
 }
 
-static int queue_userspace_packet(struct net *net, int dp_ifindex,
-                                 struct sk_buff *skb,
+static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
                                  const struct dp_upcall_info *upcall_info)
 {
        struct ovs_header *upcall;
        struct sk_buff *nskb = NULL;
        struct sk_buff *user_skb; /* to be queued to userspace */
        struct nlattr *nla;
-       int err;
+       struct genl_info info = {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
+               .dst_sk = ovs_dp_get_net(dp)->genl_sock,
+#endif
+               .snd_portid = upcall_info->portid,
+       };
+       size_t len, plen;
+       unsigned int hlen;
+       int err, dp_ifindex;
+
+       dp_ifindex = get_dpifindex(dp);
+       if (!dp_ifindex)
+               return -ENODEV;
 
        if (vlan_tx_tag_present(skb)) {
                nskb = skb_clone(skb, GFP_ATOMIC);
@@ -425,7 +427,22 @@ static int queue_userspace_packet(struct net *net, int dp_ifindex,
                goto out;
        }
 
-       user_skb = genlmsg_new(upcall_msg_size(skb, upcall_info->userdata), GFP_ATOMIC);
+       /* Complete checksum if needed */
+       if (skb->ip_summed == CHECKSUM_PARTIAL &&
+           (err = skb_checksum_help(skb)))
+               goto out;
+
+       /* Older versions of OVS user space enforce alignment of the last
+        * Netlink attribute to NLA_ALIGNTO which would require extensive
+        * padding logic. Only perform zerocopy if padding is not required.
+        */
+       if (dp->user_features & OVS_DP_F_UNALIGNED)
+               hlen = skb_zerocopy_headlen(skb);
+       else
+               hlen = skb->len;
+
+       len = upcall_msg_size(upcall_info->userdata, hlen);
+       user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
        if (!user_skb) {
                err = -ENOMEM;
                goto out;
@@ -444,43 +461,29 @@ static int queue_userspace_packet(struct net *net, int dp_ifindex,
                          nla_len(upcall_info->userdata),
                          nla_data(upcall_info->userdata));
 
-       nla = __nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, skb->len);
+       /* Only reserve room for attribute header, packet data is added
+        * in skb_zerocopy() */
+       if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
+               err = -ENOBUFS;
+               goto out;
+       }
+       nla->nla_len = nla_attr_size(skb->len);
+
+       skb_zerocopy(user_skb, skb, skb->len, hlen);
 
-       skb_copy_and_csum_dev(skb, nla_data(nla));
+       /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
+       if (!(dp->user_features & OVS_DP_F_UNALIGNED) &&
+           (plen = (ALIGN(user_skb->len, NLA_ALIGNTO) - user_skb->len)) > 0)
+               memset(skb_put(user_skb, plen), 0, plen);
 
-       genlmsg_end(user_skb, upcall);
-       err = genlmsg_unicast(net, user_skb, upcall_info->portid);
+       ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 
+       err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 out:
        kfree_skb(nskb);
        return err;
 }
 
-/* Called with ovs_mutex. */
-static int flush_flows(struct datapath *dp)
-{
-       struct flow_table *old_table;
-       struct flow_table *new_table;
-
-       old_table = ovsl_dereference(dp->table);
-       new_table = ovs_flow_tbl_alloc(TBL_MIN_BUCKETS);
-       if (!new_table)
-               return -ENOMEM;
-
-       rcu_assign_pointer(dp->table, new_table);
-
-       ovs_flow_tbl_destroy(old_table, true);
-       return 0;
-}
-
-static void clear_stats(struct sw_flow *flow)
-{
-       flow->used = 0;
-       flow->tcp_flags = 0;
-       flow->packet_count = 0;
-       flow->byte_count = 0;
-}
-
 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 {
        struct ovs_header *ovs_header = info->userhdr;
@@ -519,7 +522,7 @@ static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
                packet->protocol = htons(ETH_P_802_2);
 
        /* Build an sw_flow for sending this packet. */
-       flow = ovs_flow_alloc();
+       flow = ovs_flow_alloc(false);
        err = PTR_ERR(flow);
        if (IS_ERR(flow))
                goto err_kfree_skb;
@@ -585,15 +588,18 @@ static struct genl_ops dp_packet_genl_ops[] = {
        }
 };
 
-static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats)
+static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats,
+                        struct ovs_dp_megaflow_stats *mega_stats)
 {
-       struct flow_table *table;
        int i;
 
-       table = rcu_dereference_check(dp->table, lockdep_ovsl_is_held());
-       stats->n_flows = ovs_flow_tbl_count(table);
+       memset(mega_stats, 0, sizeof(*mega_stats));
+
+       stats->n_flows = ovs_flow_tbl_count(&dp->table);
+       mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 
        stats->n_hit = stats->n_missed = stats->n_lost = 0;
+
        for_each_possible_cpu(i) {
                const struct dp_stats_percpu *percpu_stats;
                struct dp_stats_percpu local_stats;
@@ -609,6 +615,7 @@ static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats)
                stats->n_hit += local_stats.n_hit;
                stats->n_missed += local_stats.n_missed;
                stats->n_lost += local_stats.n_lost;
+               mega_stats->n_mask_hit += local_stats.n_mask_hit;
        }
 }
 
@@ -651,10 +658,10 @@ static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
        const int skb_orig_len = skb->len;
        struct nlattr *start;
        struct ovs_flow_stats stats;
+       __be16 tcp_flags;
+       unsigned long used;
        struct ovs_header *ovs_header;
        struct nlattr *nla;
-       unsigned long used;
-       u8 tcp_flags;
        int err;
 
        ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
@@ -683,24 +690,17 @@ static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
 
        nla_nest_end(skb, nla);
 
-       spin_lock_bh(&flow->lock);
-       used = flow->used;
-       stats.n_packets = flow->packet_count;
-       stats.n_bytes = flow->byte_count;
-       tcp_flags = flow->tcp_flags;
-       spin_unlock_bh(&flow->lock);
-
+       ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
        if (used &&
            nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
                goto nla_put_failure;
 
        if (stats.n_packets &&
-           nla_put(skb, OVS_FLOW_ATTR_STATS,
-                   sizeof(struct ovs_flow_stats), &stats))
+           nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
                goto nla_put_failure;
 
-       if (tcp_flags &&
-           nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, tcp_flags))
+       if ((u8)ntohs(tcp_flags) &&
+            nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
                goto nla_put_failure;
 
        /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
@@ -717,8 +717,7 @@ static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
        if (start) {
                const struct sw_flow_actions *sf_acts;
 
-               sf_acts = rcu_dereference_check(flow->sf_acts,
-                                               lockdep_ovsl_is_held());
+               sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 
                err = ovs_nla_put_actions(sf_acts->actions,
                                          sf_acts->actions_len, skb);
@@ -742,27 +741,30 @@ error:
        return err;
 }
 
-static struct sk_buff *ovs_flow_cmd_alloc_info(struct sw_flow *flow)
+static struct sk_buff *ovs_flow_cmd_alloc_info(struct sw_flow *flow,
+                                              struct genl_info *info)
 {
-       const struct sw_flow_actions *sf_acts;
+       size_t len;
 
-       sf_acts = ovsl_dereference(flow->sf_acts);
+       len = ovs_flow_cmd_msg_size(ovsl_dereference(flow->sf_acts));
 
-       return genlmsg_new(ovs_flow_cmd_msg_size(sf_acts), GFP_KERNEL);
+       return genlmsg_new_unicast(len, info, GFP_KERNEL);
 }
 
 static struct sk_buff *ovs_flow_cmd_build_info(struct sw_flow *flow,
                                               struct datapath *dp,
-                                              u32 portid, u32 seq, u8 cmd)
+                                              struct genl_info *info,
+                                              u8 cmd)
 {
        struct sk_buff *skb;
        int retval;
 
-       skb = ovs_flow_cmd_alloc_info(flow);
+       skb = ovs_flow_cmd_alloc_info(flow, info);
        if (!skb)
                return ERR_PTR(-ENOMEM);
 
-       retval = ovs_flow_cmd_fill_info(flow, dp, skb, portid, seq, 0, cmd);
+       retval = ovs_flow_cmd_fill_info(flow, dp, skb, info->snd_portid,
+                                       info->snd_seq, 0, cmd);
        BUG_ON(retval < 0);
        return skb;
 }
@@ -776,9 +778,9 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
        struct sw_flow_mask mask;
        struct sk_buff *reply;
        struct datapath *dp;
-       struct flow_table *table;
        struct sw_flow_actions *acts = NULL;
        struct sw_flow_match match;
+       bool exact_5tuple;
        int error;
 
        /* Extract key. */
@@ -787,7 +789,7 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
                goto error;
 
        ovs_match_init(&match, &key, &mask);
-       error = ovs_nla_get_match(&match,
+       error = ovs_nla_get_match(&match, &exact_5tuple,
                                  a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
        if (error)
                goto error;
@@ -817,64 +819,33 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
        if (!dp)
                goto err_unlock_ovs;
 
-       table = ovsl_dereference(dp->table);
-
        /* Check if this is a duplicate flow */
-       flow = ovs_flow_tbl_lookup(table, &key);
+       flow = ovs_flow_tbl_lookup(&dp->table, &key);
        if (!flow) {
-               struct flow_table *new_table = NULL;
-               struct sw_flow_mask *mask_p;
-
                /* Bail out if we're not allowed to create a new flow. */
                error = -ENOENT;
                if (info->genlhdr->cmd == OVS_FLOW_CMD_SET)
                        goto err_unlock_ovs;
 
-               /* Expand table, if necessary, to make room. */
-               if (ovs_flow_tbl_need_to_expand(table))
-                       new_table = ovs_flow_tbl_expand(table);
-               else if (time_after(jiffies, dp->last_rehash + REHASH_FLOW_INTERVAL))
-                       new_table = ovs_flow_tbl_rehash(table);
-
-               if (new_table && !IS_ERR(new_table)) {
-                       rcu_assign_pointer(dp->table, new_table);
-                       ovs_flow_tbl_destroy(table, true);
-                       table = ovsl_dereference(dp->table);
-                       dp->last_rehash = jiffies;
-               }
-
                /* Allocate flow. */
-               flow = ovs_flow_alloc();
+               flow = ovs_flow_alloc(!exact_5tuple);
                if (IS_ERR(flow)) {
                        error = PTR_ERR(flow);
                        goto err_unlock_ovs;
                }
-               clear_stats(flow);
 
                flow->key = masked_key;
                flow->unmasked_key = key;
-
-               /* Make sure mask is unique in the system */
-               mask_p = ovs_sw_flow_mask_find(table, &mask);
-               if (!mask_p) {
-                       /* Allocate a new mask if none exsits. */
-                       mask_p = ovs_sw_flow_mask_alloc();
-                       if (!mask_p)
-                               goto err_flow_free;
-                       mask_p->key = mask.key;
-                       mask_p->range = mask.range;
-                       ovs_sw_flow_mask_insert(table, mask_p);
-               }
-
-               ovs_sw_flow_mask_add_ref(mask_p);
-               flow->mask = mask_p;
                rcu_assign_pointer(flow->sf_acts, acts);
 
                /* Put flow in bucket. */
-               ovs_flow_tbl_insert(table, flow);
+               error = ovs_flow_tbl_insert(&dp->table, flow, &mask);
+               if (error) {
+                       acts = NULL;
+                       goto err_flow_free;
+               }
 
-               reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
-                                               info->snd_seq, OVS_FLOW_CMD_NEW);
+               reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
        } else {
                /* We found a matching flow. */
                struct sw_flow_actions *old_acts;
@@ -902,15 +873,11 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
                rcu_assign_pointer(flow->sf_acts, acts);
                ovs_nla_free_flow_actions(old_acts);
 
-               reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
-                                              info->snd_seq, OVS_FLOW_CMD_NEW);
+               reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
 
                /* Clear stats. */
-               if (a[OVS_FLOW_ATTR_CLEAR]) {
-                       spin_lock_bh(&flow->lock);
-                       clear_stats(flow);
-                       spin_unlock_bh(&flow->lock);
-               }
+               if (a[OVS_FLOW_ATTR_CLEAR])
+                       ovs_flow_stats_clear(flow);
        }
        ovs_unlock();
 
@@ -939,7 +906,6 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
        struct sk_buff *reply;
        struct sw_flow *flow;
        struct datapath *dp;
-       struct flow_table *table;
        struct sw_flow_match match;
        int err;
 
@@ -949,7 +915,7 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
        }
 
        ovs_match_init(&match, &key, NULL);
-       err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
+       err = ovs_nla_get_match(&match, NULL, a[OVS_FLOW_ATTR_KEY], NULL);
        if (err)
                return err;
 
@@ -960,15 +926,13 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
                goto unlock;
        }
 
-       table = ovsl_dereference(dp->table);
-       flow = ovs_flow_tbl_lookup(table, &key);
+       flow = ovs_flow_tbl_lookup(&dp->table, &key);
        if (!flow || !ovs_flow_cmp_unmasked_key(flow, &match)) {
                err = -ENOENT;
                goto unlock;
        }
 
-       reply = ovs_flow_cmd_build_info(flow, dp, info->snd_portid,
-                                       info->snd_seq, OVS_FLOW_CMD_NEW);
+       reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
        if (IS_ERR(reply)) {
                err = PTR_ERR(reply);
                goto unlock;
@@ -989,7 +953,6 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
        struct sk_buff *reply;
        struct sw_flow *flow;
        struct datapath *dp;
-       struct flow_table *table;
        struct sw_flow_match match;
        int err;
 
@@ -1001,29 +964,28 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
        }
 
        if (!a[OVS_FLOW_ATTR_KEY]) {
-               err = flush_flows(dp);
+               err = ovs_flow_tbl_flush(&dp->table);
                goto unlock;
        }
 
        ovs_match_init(&match, &key, NULL);
-       err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
+       err = ovs_nla_get_match(&match, NULL, a[OVS_FLOW_ATTR_KEY], NULL);
        if (err)
                goto unlock;
 
-       table = ovsl_dereference(dp->table);
-       flow = ovs_flow_tbl_lookup(table, &key);
+       flow = ovs_flow_tbl_lookup(&dp->table, &key);
        if (!flow || !ovs_flow_cmp_unmasked_key(flow, &match)) {
                err = -ENOENT;
                goto unlock;
        }
 
-       reply = ovs_flow_cmd_alloc_info(flow);
+       reply = ovs_flow_cmd_alloc_info(flow, info);
        if (!reply) {
                err = -ENOMEM;
                goto unlock;
        }
 
-       ovs_flow_tbl_remove(table, flow);
+       ovs_flow_tbl_remove(&dp->table, flow);
 
        err = ovs_flow_cmd_fill_info(flow, dp, reply, info->snd_portid,
                                     info->snd_seq, 0, OVS_FLOW_CMD_DEL);
@@ -1042,8 +1004,8 @@ unlock:
 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
 {
        struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
+       struct table_instance *ti;
        struct datapath *dp;
-       struct flow_table *table;
 
        rcu_read_lock();
        dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
@@ -1052,14 +1014,14 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
                return -ENODEV;
        }
 
-       table = rcu_dereference(dp->table);
+       ti = rcu_dereference(dp->table.ti);
        for (;;) {
                struct sw_flow *flow;
                u32 bucket, obj;
 
                bucket = cb->args[0];
                obj = cb->args[1];
-               flow = ovs_flow_tbl_dump_next(table, &bucket, &obj);
+               flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
                if (!flow)
                        break;
 
@@ -1103,6 +1065,7 @@ static struct genl_ops dp_flow_genl_ops[] = {
 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
        [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
        [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
+       [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
 };
 
 static struct genl_family dp_datapath_genl_family = {
@@ -1125,6 +1088,7 @@ static size_t ovs_dp_cmd_msg_size(void)
 
        msgsize += nla_total_size(IFNAMSIZ);
        msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
+       msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
 
        return msgsize;
 }
@@ -1134,6 +1098,7 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
 {
        struct ovs_header *ovs_header;
        struct ovs_dp_stats dp_stats;
+       struct ovs_dp_megaflow_stats dp_megaflow_stats;
        int err;
 
        ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
@@ -1149,8 +1114,17 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
        if (err)
                goto nla_put_failure;
 
-       get_dp_stats(dp, &dp_stats);
-       if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats), &dp_stats))
+       get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
+       if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
+                       &dp_stats))
+               goto nla_put_failure;
+
+       if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
+                       sizeof(struct ovs_dp_megaflow_stats),
+                       &dp_megaflow_stats))
+               goto nla_put_failure;
+
+       if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
                goto nla_put_failure;
 
        return genlmsg_end(skb, ovs_header);
@@ -1161,17 +1135,17 @@ error:
        return -EMSGSIZE;
 }
 
-static struct sk_buff *ovs_dp_cmd_build_info(struct datapath *dp, u32 portid,
-                                            u32 seq, u8 cmd)
+static struct sk_buff *ovs_dp_cmd_build_info(struct datapath *dp,
+                                            struct genl_info *info, u8 cmd)
 {
        struct sk_buff *skb;
        int retval;
 
-       skb = genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
+       skb = genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
        if (!skb)
                return ERR_PTR(-ENOMEM);
 
-       retval = ovs_dp_cmd_fill_info(dp, skb, portid, seq, 0, cmd);
+       retval = ovs_dp_cmd_fill_info(dp, skb, info->snd_portid, info->snd_seq, 0, cmd);
        if (retval < 0) {
                kfree_skb(skb);
                return ERR_PTR(retval);
@@ -1199,6 +1173,24 @@ static struct datapath *lookup_datapath(struct net *net,
        return dp ? dp : ERR_PTR(-ENODEV);
 }
 
+static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
+{
+       struct datapath *dp;
+
+       dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
+       if (!dp)
+               return;
+
+       WARN(dp->user_features, "Dropping previously announced user features\n");
+       dp->user_features = 0;
+}
+
+static void ovs_dp_change(struct datapath *dp, struct nlattr **a)
+{
+       if (a[OVS_DP_ATTR_USER_FEATURES])
+               dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
+}
+
 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
 {
        struct nlattr **a = info->attrs;
@@ -1223,9 +1215,8 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
        ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
 
        /* Allocate table. */
-       err = -ENOMEM;
-       rcu_assign_pointer(dp->table, ovs_flow_tbl_alloc(TBL_MIN_BUCKETS));
-       if (!dp->table)
+       err = ovs_flow_tbl_init(&dp->table);
+       if (err)
                goto err_free_dp;
 
        dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
@@ -1252,17 +1243,27 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
        parms.port_no = OVSP_LOCAL;
        parms.upcall_portid = nla_get_u32(a[OVS_DP_ATTR_UPCALL_PID]);
 
+       ovs_dp_change(dp, a);
+
        vport = new_vport(&parms);
        if (IS_ERR(vport)) {
                err = PTR_ERR(vport);
                if (err == -EBUSY)
                        err = -EEXIST;
 
+               if (err == -EEXIST) {
+                       /* An outdated user space instance that does not understand
+                        * the concept of user_features has attempted to create a new
+                        * datapath and is likely to reuse it. Drop all user features.
+                        */
+                       if (info->genlhdr->version < OVS_DP_VER_FEATURES)
+                               ovs_dp_reset_user_features(skb, info);
+               }
+
                goto err_destroy_ports_array;
        }
 
-       reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
-                                     info->snd_seq, OVS_DP_CMD_NEW);
+       reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
        err = PTR_ERR(reply);
        if (IS_ERR(reply))
                goto err_destroy_local_port;
@@ -1282,7 +1283,7 @@ err_destroy_ports_array:
 err_destroy_percpu:
        free_percpu(dp->stats_percpu);
 err_destroy_table:
-       ovs_flow_tbl_destroy(ovsl_dereference(dp->table), false);
+       ovs_flow_tbl_destroy(&dp->table);
 err_free_dp:
        release_net(ovs_dp_get_net(dp));
        kfree(dp);
@@ -1328,8 +1329,7 @@ static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
        if (IS_ERR(dp))
                goto unlock;
 
-       reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
-                                     info->snd_seq, OVS_DP_CMD_DEL);
+       reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_DEL);
        err = PTR_ERR(reply);
        if (IS_ERR(reply))
                goto unlock;
@@ -1357,8 +1357,9 @@ static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
        if (IS_ERR(dp))
                goto unlock;
 
-       reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
-                                     info->snd_seq, OVS_DP_CMD_NEW);
+       ovs_dp_change(dp, info->attrs);
+
+       reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
        if (IS_ERR(reply)) {
                err = PTR_ERR(reply);
                netlink_set_err(sock_net(skb->sk)->genl_sock, 0,
@@ -1389,8 +1390,7 @@ static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
                goto unlock;
        }
 
-       reply = ovs_dp_cmd_build_info(dp, info->snd_portid,
-                                     info->snd_seq, OVS_DP_CMD_NEW);
+       reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
        if (IS_ERR(reply)) {
                err = PTR_ERR(reply);
                goto unlock;