datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2014 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <net/genetlink.h>
  52 #include <net/net_namespace.h>
  53 #include <net/netns/generic.h>
  54
  55 #include "datapath.h"
  56 #include "flow.h"
  57 #include "flow_table.h"
  58 #include "flow_netlink.h"
  59 #include "vlan.h"
  60 #include "vport-internal_dev.h"
  61 #include "vport-netdev.h"
  62
  63 int ovs_net_id __read_mostly;
  64 EXPORT_SYMBOL_GPL(ovs_net_id);
  65
  66 static struct genl_family dp_packet_genl_family;
  67 static struct genl_family dp_flow_genl_family;
  68 static struct genl_family dp_datapath_genl_family;
  69
  70 static const struct nla_policy flow_policy[];
  71
  72 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
  73         .name = OVS_FLOW_MCGROUP
  74 };
  75
  76 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  77         .name = OVS_DATAPATH_MCGROUP
  78 };
  79
  80 struct genl_multicast_group ovs_dp_vport_multicast_group = {
  81         .name = OVS_VPORT_MCGROUP
  82 };
  83
  84 /* Check if need to build a reply message.
  85  * OVS userspace sets the NLM_F_ECHO flag if it needs the reply.
  86  */
  87 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
  88                             unsigned int group)
  89 {
  90         return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  91                genl_has_listeners(family, genl_info_net(info), group);
  92 }
  93
  94 static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
  95                        struct sk_buff *skb, struct genl_info *info)
  96 {
  97         genl_notify(family, skb, genl_info_net(info),
  98                     info->snd_portid, GROUP_ID(grp), info->nlhdr, GFP_KERNEL);
  99 }
 100
 101 /**
 102  * DOC: Locking:
 103  *
 104  * All writes e.g. Writes to device state (add/remove datapath, port, set
 105  * operations on vports, etc.), Writes to other state (flow table
 106  * modifications, set miscellaneous datapath parameters, etc.) are protected
 107  * by ovs_lock.
 108  *
 109  * Reads are protected by RCU.
 110  *
 111  * There are a few special cases (mostly stats) that have their own
 112  * synchronization but they nest under all of above and don't interact with
 113  * each other.
 114  *
 115  * The RTNL lock nests inside ovs_mutex.
 116  */
 117
 118 static DEFINE_MUTEX(ovs_mutex);
 119
 120 void ovs_lock(void)
 121 {
 122         mutex_lock(&ovs_mutex);
 123 }
 124
 125 void ovs_unlock(void)
 126 {
 127         mutex_unlock(&ovs_mutex);
 128 }
 129
 130 #ifdef CONFIG_LOCKDEP
 131 int lockdep_ovsl_is_held(void)
 132 {
 133         if (debug_locks)
 134                 return lockdep_is_held(&ovs_mutex);
 135         else
 136                 return 1;
 137 }
 138 EXPORT_SYMBOL_GPL(lockdep_ovsl_is_held);
 139 #endif
 140
 141 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 142                              const struct sw_flow_key *,
 143                              const struct dp_upcall_info *);
 144 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 145                                   const struct sw_flow_key *,
 146                                   const struct dp_upcall_info *);
 147
 148 /* Must be called with rcu_read_lock. */
 149 static struct datapath *get_dp_rcu(struct net *net, int dp_ifindex)
 150 {
 151         struct net_device *dev = dev_get_by_index_rcu(net, dp_ifindex);
 152
 153         if (dev) {
 154                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 155                 if (vport)
 156                         return vport->dp;
 157         }
 158
 159         return NULL;
 160 }
 161
 162 /* The caller must hold either ovs_mutex or rcu_read_lock to keep the
 163  * returned dp pointer valid.
 164  */
 165 static inline struct datapath *get_dp(struct net *net, int dp_ifindex)
 166 {
 167         struct datapath *dp;
 168
 169         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_ovsl_is_held());
 170         rcu_read_lock();
 171         dp = get_dp_rcu(net, dp_ifindex);
 172         rcu_read_unlock();
 173
 174         return dp;
 175 }
 176
 177 /* Must be called with rcu_read_lock or ovs_mutex. */
 178 const char *ovs_dp_name(const struct datapath *dp)
 179 {
 180         struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 181         return vport->ops->get_name(vport);
 182 }
 183
 184 static int get_dpifindex(const struct datapath *dp)
 185 {
 186         struct vport *local;
 187         int ifindex;
 188
 189         rcu_read_lock();
 190
 191         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 192         if (local)
 193                 ifindex = netdev_vport_priv(local)->dev->ifindex;
 194         else
 195                 ifindex = 0;
 196
 197         rcu_read_unlock();
 198
 199         return ifindex;
 200 }
 201
 202 static void destroy_dp_rcu(struct rcu_head *rcu)
 203 {
 204         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 205
 206         ovs_flow_tbl_destroy(&dp->table);
 207         free_percpu(dp->stats_percpu);
 208         release_net(ovs_dp_get_net(dp));
 209         kfree(dp->ports);
 210         kfree(dp);
 211 }
 212
 213 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 214                                             u16 port_no)
 215 {
 216         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 217 }
 218
 219 /* Called with ovs_mutex or RCU read lock. */
 220 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 221 {
 222         struct vport *vport;
 223         struct hlist_head *head;
 224
 225         head = vport_hash_bucket(dp, port_no);
 226         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 227                 if (vport->port_no == port_no)
 228                         return vport;
 229         }
 230         return NULL;
 231 }
 232
 233 /* Called with ovs_mutex. */
 234 static struct vport *new_vport(const struct vport_parms *parms)
 235 {
 236         struct vport *vport;
 237
 238         vport = ovs_vport_add(parms);
 239         if (!IS_ERR(vport)) {
 240                 struct datapath *dp = parms->dp;
 241                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 242
 243                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 244         }
 245         return vport;
 246 }
 247
 248 void ovs_dp_detach_port(struct vport *p)
 249 {
 250         ASSERT_OVSL();
 251
 252         /* First drop references to device. */
 253         hlist_del_rcu(&p->dp_hash_node);
 254
 255         /* Then destroy it. */
 256         ovs_vport_del(p);
 257 }
 258
 259 /* Must be called with rcu_read_lock. */
 260 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
 261 {
 262         const struct vport *p = OVS_CB(skb)->input_vport;
 263         struct datapath *dp = p->dp;
 264         struct sw_flow *flow;
 265         struct sw_flow_actions *sf_acts;
 266         struct dp_stats_percpu *stats;
 267         u64 *stats_counter;
 268         u32 n_mask_hit;
 269
 270         stats = this_cpu_ptr(dp->stats_percpu);
 271
 272         /* Look up flow. */
 273         flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
 274                                          &n_mask_hit);
 275         if (unlikely(!flow)) {
 276                 struct dp_upcall_info upcall;
 277                 int error;
 278
 279                 upcall.cmd = OVS_PACKET_CMD_MISS;
 280                 upcall.userdata = NULL;
 281                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 282                 upcall.egress_tun_info = NULL;
 283                 error = ovs_dp_upcall(dp, skb, key, &upcall);
 284                 if (unlikely(error))
 285                         kfree_skb(skb);
 286                 else
 287                         consume_skb(skb);
 288                 stats_counter = &stats->n_missed;
 289                 goto out;
 290         }
 291
 292         ovs_flow_stats_update(flow, key->tp.flags, skb);
 293         sf_acts = rcu_dereference(flow->sf_acts);
 294         ovs_execute_actions(dp, skb, sf_acts, key);
 295
 296         stats_counter = &stats->n_hit;
 297
 298 out:
 299         /* Update datapath statistics. */
 300         u64_stats_update_begin(&stats->syncp);
 301         (*stats_counter)++;
 302         stats->n_mask_hit += n_mask_hit;
 303         u64_stats_update_end(&stats->syncp);
 304 }
 305
 306 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 307                   const struct sw_flow_key *key,
 308                   const struct dp_upcall_info *upcall_info)
 309 {
 310         struct dp_stats_percpu *stats;
 311         int err;
 312
 313         if (upcall_info->portid == 0) {
 314                 err = -ENOTCONN;
 315                 goto err;
 316         }
 317
 318         if (!skb_is_gso(skb))
 319                 err = queue_userspace_packet(dp, skb, key, upcall_info);
 320         else
 321                 err = queue_gso_packets(dp, skb, key, upcall_info);
 322         if (err)
 323                 goto err;
 324
 325         return 0;
 326
 327 err:
 328         stats = this_cpu_ptr(dp->stats_percpu);
 329
 330         u64_stats_update_begin(&stats->syncp);
 331         stats->n_lost++;
 332         u64_stats_update_end(&stats->syncp);
 333
 334         return err;
 335 }
 336
 337 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 338                              const struct sw_flow_key *key,
 339                              const struct dp_upcall_info *upcall_info)
 340 {
 341         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 342         struct sw_flow_key later_key;
 343         struct sk_buff *segs, *nskb;
 344         struct ovs_skb_cb ovs_cb;
 345         int err;
 346
 347         ovs_cb = *OVS_CB(skb);
 348         segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 349         *OVS_CB(skb) = ovs_cb;
 350         if (IS_ERR(segs))
 351                 return PTR_ERR(segs);
 352         if (segs == NULL)
 353                 return -EINVAL;
 354
 355         if (gso_type & SKB_GSO_UDP) {
 356                 /* The initial flow key extracted by ovs_flow_key_extract()
 357                  * in this case is for a first fragment, so we need to
 358                  * properly mark later fragments.
 359                  */
 360                 later_key = *key;
 361                 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 362         }
 363
 364         /* Queue all of the segments. */
 365         skb = segs;
 366         do {
 367                 *OVS_CB(skb) = ovs_cb;
 368                 if (gso_type & SKB_GSO_UDP && skb != segs)
 369                         key = &later_key;
 370
 371                 err = queue_userspace_packet(dp, skb, key, upcall_info);
 372                 if (err)
 373                         break;
 374
 375         } while ((skb = skb->next));
 376
 377         /* Free all of the segments. */
 378         skb = segs;
 379         do {
 380                 nskb = skb->next;
 381                 if (err)
 382                         kfree_skb(skb);
 383                 else
 384                         consume_skb(skb);
 385         } while ((skb = nskb));
 386         return err;
 387 }
 388
 389 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 390                               unsigned int hdrlen)
 391 {
 392         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 393                 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 394                 + nla_total_size(ovs_key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 395
 396         /* OVS_PACKET_ATTR_USERDATA */
 397         if (upcall_info->userdata)
 398                 size += NLA_ALIGN(upcall_info->userdata->nla_len);
 399
 400         /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
 401         if (upcall_info->egress_tun_info)
 402                 size += nla_total_size(ovs_tun_key_attr_size());
 403
 404         return size;
 405 }
 406
 407 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 408                                   const struct sw_flow_key *key,
 409                                   const struct dp_upcall_info *upcall_info)
 410 {
 411         struct ovs_header *upcall;
 412         struct sk_buff *nskb = NULL;
 413         struct sk_buff *user_skb = NULL; /* to be queued to userspace */
 414         struct nlattr *nla;
 415         struct genl_info info = {
 416 #ifdef HAVE_GENLMSG_NEW_UNICAST
 417                 .dst_sk = ovs_dp_get_net(dp)->genl_sock,
 418 #endif
 419                 .snd_portid = upcall_info->portid,
 420         };
 421         size_t len;
 422         unsigned int hlen;
 423         int err, dp_ifindex;
 424
 425         dp_ifindex = get_dpifindex(dp);
 426         if (!dp_ifindex)
 427                 return -ENODEV;
 428
 429         if (skb_vlan_tag_present(skb)) {
 430                 nskb = skb_clone(skb, GFP_ATOMIC);
 431                 if (!nskb)
 432                         return -ENOMEM;
 433
 434                 nskb = vlan_insert_tag_set_proto(nskb, nskb->vlan_proto, skb_vlan_tag_get(nskb));
 435                 if (!nskb)
 436                         return -ENOMEM;
 437
 438                 vlan_set_tci(nskb, 0);
 439
 440                 skb = nskb;
 441         }
 442
 443         if (nla_attr_size(skb->len) > USHRT_MAX) {
 444                 err = -EFBIG;
 445                 goto out;
 446         }
 447
 448         /* Complete checksum if needed */
 449         if (skb->ip_summed == CHECKSUM_PARTIAL &&
 450             (err = skb_checksum_help(skb)))
 451                 goto out;
 452
 453         /* Older versions of OVS user space enforce alignment of the last
 454          * Netlink attribute to NLA_ALIGNTO which would require extensive
 455          * padding logic. Only perform zerocopy if padding is not required.
 456          */
 457         if (dp->user_features & OVS_DP_F_UNALIGNED)
 458                 hlen = skb_zerocopy_headlen(skb);
 459         else
 460                 hlen = skb->len;
 461
 462         len = upcall_msg_size(upcall_info, hlen);
 463         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 464         if (!user_skb) {
 465                 err = -ENOMEM;
 466                 goto out;
 467         }
 468
 469         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 470                              0, upcall_info->cmd);
 471         upcall->dp_ifindex = dp_ifindex;
 472
 473         err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
 474         BUG_ON(err);
 475
 476         if (upcall_info->userdata)
 477                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 478                           nla_len(upcall_info->userdata),
 479                           nla_data(upcall_info->userdata));
 480
 481         if (upcall_info->egress_tun_info) {
 482                 nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_EGRESS_TUN_KEY);
 483                 err = ovs_nla_put_egress_tunnel_key(user_skb,
 484                                                     upcall_info->egress_tun_info);
 485                 BUG_ON(err);
 486                 nla_nest_end(user_skb, nla);
 487         }
 488
 489         /* Only reserve room for attribute header, packet data is added
 490          * in skb_zerocopy()
 491          */
 492         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 493                 err = -ENOBUFS;
 494                 goto out;
 495         }
 496         nla->nla_len = nla_attr_size(skb->len);
 497
 498         err = skb_zerocopy(user_skb, skb, skb->len, hlen);
 499         if (err)
 500                 goto out;
 501
 502         /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 503         if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 504                 size_t plen = NLA_ALIGN(user_skb->len) - user_skb->len;
 505
 506                 if (plen > 0)
 507                         memset(skb_put(user_skb, plen), 0, plen);
 508         }
 509
 510         ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 511
 512         err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 513         user_skb = NULL;
 514 out:
 515         if (err)
 516                 skb_tx_error(skb);
 517         kfree_skb(user_skb);
 518         kfree_skb(nskb);
 519         return err;
 520 }
 521
 522 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 523 {
 524         struct ovs_header *ovs_header = info->userhdr;
 525         struct nlattr **a = info->attrs;
 526         struct sw_flow_actions *acts;
 527         struct sk_buff *packet;
 528         struct sw_flow *flow;
 529         struct sw_flow_actions *sf_acts;
 530         struct datapath *dp;
 531         struct ethhdr *eth;
 532         struct vport *input_vport;
 533         int len;
 534         int err;
 535         bool log = !a[OVS_PACKET_ATTR_PROBE];
 536
 537         err = -EINVAL;
 538         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 539             !a[OVS_PACKET_ATTR_ACTIONS])
 540                 goto err;
 541
 542         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 543         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 544         err = -ENOMEM;
 545         if (!packet)
 546                 goto err;
 547         skb_reserve(packet, NET_IP_ALIGN);
 548
 549         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 550
 551         skb_reset_mac_header(packet);
 552         eth = eth_hdr(packet);
 553
 554         /* Normally, setting the skb 'protocol' field would be handled by a
 555          * call to eth_type_trans(), but it assumes there's a sending
 556          * device, which we may not have.
 557          */
 558         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 559                 packet->protocol = eth->h_proto;
 560         else
 561                 packet->protocol = htons(ETH_P_802_2);
 562
 563         /* Build an sw_flow for sending this packet. */
 564         flow = ovs_flow_alloc();
 565         err = PTR_ERR(flow);
 566         if (IS_ERR(flow))
 567                 goto err_kfree_skb;
 568
 569         err = ovs_flow_key_extract_userspace(a[OVS_PACKET_ATTR_KEY], packet,
 570                                              &flow->key, log);
 571         if (err)
 572                 goto err_flow_free;
 573
 574         err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
 575                                    &flow->key, &acts, log);
 576         if (err)
 577                 goto err_flow_free;
 578
 579         rcu_assign_pointer(flow->sf_acts, acts);
 580         OVS_CB(packet)->egress_tun_info = NULL;
 581         packet->priority = flow->key.phy.priority;
 582         packet->mark = flow->key.phy.skb_mark;
 583
 584         rcu_read_lock();
 585         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
 586         err = -ENODEV;
 587         if (!dp)
 588                 goto err_unlock;
 589
 590         input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 591         if (!input_vport)
 592                 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 593
 594         if (!input_vport)
 595                 goto err_unlock;
 596
 597         OVS_CB(packet)->input_vport = input_vport;
 598         sf_acts = rcu_dereference(flow->sf_acts);
 599
 600         local_bh_disable();
 601         err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
 602         local_bh_enable();
 603         rcu_read_unlock();
 604
 605         ovs_flow_free(flow, false);
 606         return err;
 607
 608 err_unlock:
 609         rcu_read_unlock();
 610 err_flow_free:
 611         ovs_flow_free(flow, false);
 612 err_kfree_skb:
 613         kfree_skb(packet);
 614 err:
 615         return err;
 616 }
 617
 618 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 619         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 620         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 621         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 622         [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
 623 };
 624
 625 static struct genl_ops dp_packet_genl_ops[] = {
 626         { .cmd = OVS_PACKET_CMD_EXECUTE,
 627           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 628           .policy = packet_policy,
 629           .doit = ovs_packet_cmd_execute
 630         }
 631 };
 632
 633 static struct genl_family dp_packet_genl_family = {
 634         .id = GENL_ID_GENERATE,
 635         .hdrsize = sizeof(struct ovs_header),
 636         .name = OVS_PACKET_FAMILY,
 637         .version = OVS_PACKET_VERSION,
 638         .maxattr = OVS_PACKET_ATTR_MAX,
 639         .netnsok = true,
 640         .parallel_ops = true,
 641         .ops = dp_packet_genl_ops,
 642         .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
 643 };
 644
 645 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
 646                          struct ovs_dp_megaflow_stats *mega_stats)
 647 {
 648         int i;
 649
 650         memset(mega_stats, 0, sizeof(*mega_stats));
 651
 652         stats->n_flows = ovs_flow_tbl_count(&dp->table);
 653         mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 654
 655         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 656
 657         for_each_possible_cpu(i) {
 658                 const struct dp_stats_percpu *percpu_stats;
 659                 struct dp_stats_percpu local_stats;
 660                 unsigned int start;
 661
 662                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 663
 664                 do {
 665                         start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
 666                         local_stats = *percpu_stats;
 667                 } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
 668
 669                 stats->n_hit += local_stats.n_hit;
 670                 stats->n_missed += local_stats.n_missed;
 671                 stats->n_lost += local_stats.n_lost;
 672                 mega_stats->n_mask_hit += local_stats.n_mask_hit;
 673         }
 674 }
 675
 676 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
 677 {
 678         return ovs_identifier_is_ufid(sfid) &&
 679                !(ufid_flags & OVS_UFID_F_OMIT_KEY);
 680 }
 681
 682 static bool should_fill_mask(uint32_t ufid_flags)
 683 {
 684         return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
 685 }
 686
 687 static bool should_fill_actions(uint32_t ufid_flags)
 688 {
 689         return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
 690 }
 691
 692 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
 693                                     const struct sw_flow_id *sfid,
 694                                     uint32_t ufid_flags)
 695 {
 696         size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
 697
 698         /* OVS_FLOW_ATTR_UFID */
 699         if (sfid && ovs_identifier_is_ufid(sfid))
 700                 len += nla_total_size(sfid->ufid_len);
 701
 702         /* OVS_FLOW_ATTR_KEY */
 703         if (!sfid || should_fill_key(sfid, ufid_flags))
 704                 len += nla_total_size(ovs_key_attr_size());
 705
 706         /* OVS_FLOW_ATTR_MASK */
 707         if (should_fill_mask(ufid_flags))
 708                 len += nla_total_size(ovs_key_attr_size());
 709
 710         /* OVS_FLOW_ATTR_ACTIONS */
 711         if (should_fill_actions(ufid_flags))
 712                 len += nla_total_size(acts->actions_len);
 713
 714         return len
 715                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 716                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 717                 + nla_total_size(8); /* OVS_FLOW_ATTR_USED */
 718 }
 719
 720 /* Called with ovs_mutex or RCU read lock. */
 721 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
 722                                    struct sk_buff *skb)
 723 {
 724         struct ovs_flow_stats stats;
 725         __be16 tcp_flags;
 726         unsigned long used;
 727
 728         ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 729
 730         if (used &&
 731             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
 732                 return -EMSGSIZE;
 733
 734         if (stats.n_packets &&
 735             nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
 736                 return -EMSGSIZE;
 737
 738         if ((u8)ntohs(tcp_flags) &&
 739              nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 740                 return -EMSGSIZE;
 741
 742         return 0;
 743 }
 744
 745 /* Called with ovs_mutex or RCU read lock. */
 746 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
 747                                      struct sk_buff *skb, int skb_orig_len)
 748 {
 749         struct nlattr *start;
 750         int err;
 751
 752         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 753          * this is the first flow to be dumped into 'skb'.  This is unusual for
 754          * Netlink but individual action lists can be longer than
 755          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 756          * The userspace caller can always fetch the actions separately if it
 757          * really wants them.  (Most userspace callers in fact don't care.)
 758          *
 759          * This can only fail for dump operations because the skb is always
 760          * properly sized for single flows.
 761          */
 762         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 763         if (start) {
 764                 const struct sw_flow_actions *sf_acts;
 765
 766                 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 767                 err = ovs_nla_put_actions(sf_acts->actions,
 768                                           sf_acts->actions_len, skb);
 769
 770                 if (!err)
 771                         nla_nest_end(skb, start);
 772                 else {
 773                         if (skb_orig_len)
 774                                 return err;
 775
 776                         nla_nest_cancel(skb, start);
 777                 }
 778         } else if (skb_orig_len) {
 779                 return -EMSGSIZE;
 780         }
 781
 782         return 0;
 783 }
 784
 785 /* Called with ovs_mutex or RCU read lock. */
 786 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
 787                                   struct sk_buff *skb, u32 portid,
 788                                   u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
 789 {
 790         const int skb_orig_len = skb->len;
 791         struct ovs_header *ovs_header;
 792         int err;
 793
 794         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
 795                                  flags, cmd);
 796         if (!ovs_header)
 797                 return -EMSGSIZE;
 798
 799         ovs_header->dp_ifindex = dp_ifindex;
 800
 801         err = ovs_nla_put_identifier(flow, skb);
 802         if (err)
 803                 goto error;
 804
 805         if (should_fill_key(&flow->id, ufid_flags)) {
 806                 err = ovs_nla_put_masked_key(flow, skb);
 807                 if (err)
 808                         goto error;
 809         }
 810
 811         if (should_fill_mask(ufid_flags)) {
 812                 err = ovs_nla_put_mask(flow, skb);
 813                 if (err)
 814                         goto error;
 815         }
 816
 817         err = ovs_flow_cmd_fill_stats(flow, skb);
 818         if (err)
 819                 goto error;
 820
 821         if (should_fill_actions(ufid_flags)) {
 822                 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
 823                 if (err)
 824                         goto error;
 825         }
 826
 827         genlmsg_end(skb, ovs_header);
 828         return 0;
 829
 830 error:
 831         genlmsg_cancel(skb, ovs_header);
 832         return err;
 833 }
 834
 835 /* May not be called with RCU read lock. */
 836 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 837                                                const struct sw_flow_id *sfid,
 838                                                struct genl_info *info,
 839                                                bool always,
 840                                                uint32_t ufid_flags)
 841 {
 842         struct sk_buff *skb;
 843         size_t len;
 844
 845         if (!always && !ovs_must_notify(&dp_flow_genl_family, info,
 846                                         GROUP_ID(&ovs_dp_flow_multicast_group)))
 847                 return NULL;
 848
 849         len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
 850         skb = genlmsg_new_unicast(len, info, GFP_KERNEL);
 851         if (!skb)
 852                 return ERR_PTR(-ENOMEM);
 853
 854         return skb;
 855 }
 856
 857 /* Called with ovs_mutex. */
 858 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
 859                                                int dp_ifindex,
 860                                                struct genl_info *info, u8 cmd,
 861                                                bool always, u32 ufid_flags)
 862 {
 863         struct sk_buff *skb;
 864         int retval;
 865
 866         skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
 867                                       &flow->id, info, always, ufid_flags);
 868         if (IS_ERR_OR_NULL(skb))
 869                 return skb;
 870
 871         retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
 872                                         info->snd_portid, info->snd_seq, 0,
 873                                         cmd, ufid_flags);
 874         BUG_ON(retval < 0);
 875         return skb;
 876 }
 877
 878 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 879 {
 880         struct nlattr **a = info->attrs;
 881         struct ovs_header *ovs_header = info->userhdr;
 882         struct sw_flow *flow = NULL, *new_flow;
 883         struct sw_flow_mask mask;
 884         struct sk_buff *reply;
 885         struct datapath *dp;
 886         struct sw_flow_key key;
 887         struct sw_flow_actions *acts;
 888         struct sw_flow_match match;
 889         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
 890         int error;
 891         bool log = !a[OVS_FLOW_ATTR_PROBE];
 892
 893         /* Must have key and actions. */
 894         error = -EINVAL;
 895         if (!a[OVS_FLOW_ATTR_KEY]) {
 896                 OVS_NLERR(log, "Flow key attr not present in new flow.");
 897                 goto error;
 898         }
 899         if (!a[OVS_FLOW_ATTR_ACTIONS]) {
 900                 OVS_NLERR(log, "Flow actions attr not present in new flow.");
 901                 goto error;
 902         }
 903
 904         /* Most of the time we need to allocate a new flow, do it before
 905          * locking.
 906          */
 907         new_flow = ovs_flow_alloc();
 908         if (IS_ERR(new_flow)) {
 909                 error = PTR_ERR(new_flow);
 910                 goto error;
 911         }
 912
 913         /* Extract key. */
 914         ovs_match_init(&match, &key, &mask);
 915         error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
 916                                   a[OVS_FLOW_ATTR_MASK], log);
 917         if (error)
 918                 goto err_kfree_flow;
 919
 920         ovs_flow_mask_key(&new_flow->key, &key, &mask);
 921
 922         /* Extract flow identifier. */
 923         error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
 924                                        &key, log);
 925         if (error)
 926                 goto err_kfree_flow;
 927
 928         /* Validate actions. */
 929         error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &new_flow->key,
 930                                      &acts, log);
 931         if (error) {
 932                 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
 933                 goto err_kfree_flow;
 934         }
 935
 936         reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
 937                                         ufid_flags);
 938         if (IS_ERR(reply)) {
 939                 error = PTR_ERR(reply);
 940                 goto err_kfree_acts;
 941         }
 942
 943         ovs_lock();
 944         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 945         if (unlikely(!dp)) {
 946                 error = -ENODEV;
 947                 goto err_unlock_ovs;
 948         }
 949
 950         /* Check if this is a duplicate flow */
 951         if (ovs_identifier_is_ufid(&new_flow->id))
 952                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
 953         if (!flow)
 954                 flow = ovs_flow_tbl_lookup(&dp->table, &key);
 955         if (likely(!flow)) {
 956                 rcu_assign_pointer(new_flow->sf_acts, acts);
 957
 958                 /* Put flow in bucket. */
 959                 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
 960                 if (unlikely(error)) {
 961                         acts = NULL;
 962                         goto err_unlock_ovs;
 963                 }
 964
 965                 if (unlikely(reply)) {
 966                         error = ovs_flow_cmd_fill_info(new_flow,
 967                                                        ovs_header->dp_ifindex,
 968                                                        reply, info->snd_portid,
 969                                                        info->snd_seq, 0,
 970                                                        OVS_FLOW_CMD_NEW,
 971                                                        ufid_flags);
 972                         BUG_ON(error < 0);
 973                 }
 974                 ovs_unlock();
 975         } else {
 976                 struct sw_flow_actions *old_acts;
 977
 978                 /* Bail out if we're not allowed to modify an existing flow.
 979                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 980                  * because Generic Netlink treats the latter as a dump
 981                  * request.  We also accept NLM_F_EXCL in case that bug ever
 982                  * gets fixed.
 983                  */
 984                 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
 985                                                          | NLM_F_EXCL))) {
 986                         error = -EEXIST;
 987                         goto err_unlock_ovs;
 988                 }
 989                 /* The flow identifier has to be the same for flow updates.
 990                  * Look for any overlapping flow.
 991                  */
 992                 if (unlikely(!ovs_flow_cmp(flow, &match))) {
 993                         if (ovs_identifier_is_key(&flow->id))
 994                                 flow = ovs_flow_tbl_lookup_exact(&dp->table,
 995                                                                  &match);
 996                         else /* UFID matches but key is different */
 997                                 flow = NULL;
 998                         if (!flow) {
 999                                 error = -ENOENT;
1000                                 goto err_unlock_ovs;
1001                         }
1002                 }
1003                 /* Update actions. */
1004                 old_acts = ovsl_dereference(flow->sf_acts);
1005                 rcu_assign_pointer(flow->sf_acts, acts);
1006
1007                 if (unlikely(reply)) {
1008                         error = ovs_flow_cmd_fill_info(flow,
1009                                                        ovs_header->dp_ifindex,
1010                                                        reply, info->snd_portid,
1011                                                        info->snd_seq, 0,
1012                                                        OVS_FLOW_CMD_NEW,
1013                                                        ufid_flags);
1014                         BUG_ON(error < 0);
1015                 }
1016                 ovs_unlock();
1017
1018                 ovs_nla_free_flow_actions(old_acts);
1019                 ovs_flow_free(new_flow, false);
1020         }
1021
1022         if (reply)
1023                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1024         return 0;
1025
1026 err_unlock_ovs:
1027         ovs_unlock();
1028         kfree_skb(reply);
1029 err_kfree_acts:
1030         kfree(acts);
1031 err_kfree_flow:
1032         ovs_flow_free(new_flow, false);
1033 error:
1034         return error;
1035 }
1036
1037 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1038 static struct sw_flow_actions *get_flow_actions(const struct nlattr *a,
1039                                                 const struct sw_flow_key *key,
1040                                                 const struct sw_flow_mask *mask,
1041                                                 bool log)
1042 {
1043         struct sw_flow_actions *acts;
1044         struct sw_flow_key masked_key;
1045         int error;
1046
1047         ovs_flow_mask_key(&masked_key, key, mask);
1048         error = ovs_nla_copy_actions(a, &masked_key, &acts, log);
1049         if (error) {
1050                 OVS_NLERR(log,
1051                           "Actions may not be safe on all matching packets");
1052                 return ERR_PTR(error);
1053         }
1054
1055         return acts;
1056 }
1057
1058 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1059 {
1060         struct nlattr **a = info->attrs;
1061         struct ovs_header *ovs_header = info->userhdr;
1062         struct sw_flow_key key;
1063         struct sw_flow *flow;
1064         struct sw_flow_mask mask;
1065         struct sk_buff *reply = NULL;
1066         struct datapath *dp;
1067         struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1068         struct sw_flow_match match;
1069         struct sw_flow_id sfid;
1070         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1071         int error;
1072         bool log = !a[OVS_FLOW_ATTR_PROBE];
1073         bool ufid_present;
1074
1075         /* Extract key. */
1076         error = -EINVAL;
1077         if (!a[OVS_FLOW_ATTR_KEY]) {
1078                 OVS_NLERR(log, "Flow key attribute not present in set flow.");
1079                 goto error;
1080         }
1081
1082         ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1083         ovs_match_init(&match, &key, &mask);
1084         error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
1085                                   a[OVS_FLOW_ATTR_MASK], log);
1086         if (error)
1087                 goto error;
1088
1089         /* Validate actions. */
1090         if (a[OVS_FLOW_ATTR_ACTIONS]) {
1091                 acts = get_flow_actions(a[OVS_FLOW_ATTR_ACTIONS], &key, &mask,
1092                                         log);
1093                 if (IS_ERR(acts)) {
1094                         error = PTR_ERR(acts);
1095                         goto error;
1096                 }
1097
1098                 /* Can allocate before locking if have acts. */
1099                 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1100                                                 ufid_flags);
1101                 if (IS_ERR(reply)) {
1102                         error = PTR_ERR(reply);
1103                         goto err_kfree_acts;
1104                 }
1105         }
1106
1107         ovs_lock();
1108         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1109         if (unlikely(!dp)) {
1110                 error = -ENODEV;
1111                 goto err_unlock_ovs;
1112         }
1113         /* Check that the flow exists. */
1114         if (ufid_present)
1115                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1116         else
1117                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1118         if (unlikely(!flow)) {
1119                 error = -ENOENT;
1120                 goto err_unlock_ovs;
1121         }
1122
1123         /* Update actions, if present. */
1124         if (likely(acts)) {
1125                 old_acts = ovsl_dereference(flow->sf_acts);
1126                 rcu_assign_pointer(flow->sf_acts, acts);
1127
1128                 if (unlikely(reply)) {
1129                         error = ovs_flow_cmd_fill_info(flow,
1130                                                        ovs_header->dp_ifindex,
1131                                                        reply, info->snd_portid,
1132                                                        info->snd_seq, 0,
1133                                                        OVS_FLOW_CMD_NEW,
1134                                                        ufid_flags);
1135                         BUG_ON(error < 0);
1136                 }
1137         } else {
1138                 /* Could not alloc without acts before locking. */
1139                 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1140                                                 info, OVS_FLOW_CMD_NEW, false,
1141                                                 ufid_flags);
1142
1143                 if (unlikely(IS_ERR(reply))) {
1144                         error = PTR_ERR(reply);
1145                         goto err_unlock_ovs;
1146                 }
1147         }
1148
1149         /* Clear stats. */
1150         if (a[OVS_FLOW_ATTR_CLEAR])
1151                 ovs_flow_stats_clear(flow);
1152         ovs_unlock();
1153
1154         if (reply)
1155                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1156         if (old_acts)
1157                 ovs_nla_free_flow_actions(old_acts);
1158
1159         return 0;
1160
1161 err_unlock_ovs:
1162         ovs_unlock();
1163         kfree_skb(reply);
1164 err_kfree_acts:
1165         kfree(acts);
1166 error:
1167         return error;
1168 }
1169
1170 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1171 {
1172         struct nlattr **a = info->attrs;
1173         struct ovs_header *ovs_header = info->userhdr;
1174         struct sw_flow_key key;
1175         struct sk_buff *reply;
1176         struct sw_flow *flow;
1177         struct datapath *dp;
1178         struct sw_flow_match match;
1179         struct sw_flow_id ufid;
1180         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1181         int err = 0;
1182         bool log = !a[OVS_FLOW_ATTR_PROBE];
1183         bool ufid_present;
1184
1185         ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1186         if (a[OVS_FLOW_ATTR_KEY]) {
1187                 ovs_match_init(&match, &key, NULL);
1188                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
1189                                         log);
1190         } else if (!ufid_present) {
1191                 OVS_NLERR(log,
1192                           "Flow get message rejected, Key attribute missing.");
1193                 err = -EINVAL;
1194         }
1195         if (err)
1196                 return err;
1197
1198         ovs_lock();
1199         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1200         if (!dp) {
1201                 err = -ENODEV;
1202                 goto unlock;
1203         }
1204
1205         if (ufid_present)
1206                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1207         else
1208                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1209         if (!flow) {
1210                 err = -ENOENT;
1211                 goto unlock;
1212         }
1213
1214         reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1215                                         OVS_FLOW_CMD_NEW, true, ufid_flags);
1216         if (IS_ERR(reply)) {
1217                 err = PTR_ERR(reply);
1218                 goto unlock;
1219         }
1220
1221         ovs_unlock();
1222         return genlmsg_reply(reply, info);
1223 unlock:
1224         ovs_unlock();
1225         return err;
1226 }
1227
1228 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1229 {
1230         struct nlattr **a = info->attrs;
1231         struct ovs_header *ovs_header = info->userhdr;
1232         struct sw_flow_key key;
1233         struct sk_buff *reply;
1234         struct sw_flow *flow = NULL;
1235         struct datapath *dp;
1236         struct sw_flow_match match;
1237         struct sw_flow_id ufid;
1238         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1239         int err;
1240         bool log = !a[OVS_FLOW_ATTR_PROBE];
1241         bool ufid_present;
1242
1243         ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1244         if (a[OVS_FLOW_ATTR_KEY]) {
1245                 ovs_match_init(&match, &key, NULL);
1246                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
1247                                         log);
1248                 if (unlikely(err))
1249                         return err;
1250         }
1251
1252         ovs_lock();
1253         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1254         if (unlikely(!dp)) {
1255                 err = -ENODEV;
1256                 goto unlock;
1257         }
1258
1259         if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1260                 err = ovs_flow_tbl_flush(&dp->table);
1261                 goto unlock;
1262         }
1263
1264         if (ufid_present)
1265                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1266         else
1267                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1268         if (unlikely(!flow)) {
1269                 err = -ENOENT;
1270                 goto unlock;
1271         }
1272
1273         ovs_flow_tbl_remove(&dp->table, flow);
1274         ovs_unlock();
1275
1276         reply = ovs_flow_cmd_alloc_info(rcu_dereference_raw(flow->sf_acts),
1277                                         &flow->id, info, false, ufid_flags);
1278
1279         if (likely(reply)) {
1280                 if (likely(!IS_ERR(reply))) {
1281                         rcu_read_lock();        /*To keep RCU checker happy. */
1282                         err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1283                                                      reply, info->snd_portid,
1284                                                      info->snd_seq, 0,
1285                                                      OVS_FLOW_CMD_DEL,
1286                                                      ufid_flags);
1287                         rcu_read_unlock();
1288                         BUG_ON(err < 0);
1289                         ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1290                 } else {
1291                         genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1292                                      GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1293
1294                 }
1295         }
1296
1297         ovs_flow_free(flow, true);
1298         return 0;
1299 unlock:
1300         ovs_unlock();
1301         return err;
1302 }
1303
1304 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1305 {
1306         struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1307         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1308         struct table_instance *ti;
1309         struct datapath *dp;
1310         u32 ufid_flags;
1311         int err;
1312
1313         err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
1314                             OVS_FLOW_ATTR_MAX, flow_policy);
1315         if (err)
1316                 return err;
1317         ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1318
1319         rcu_read_lock();
1320         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1321         if (!dp) {
1322                 rcu_read_unlock();
1323                 return -ENODEV;
1324         }
1325
1326         ti = rcu_dereference(dp->table.ti);
1327         for (;;) {
1328                 struct sw_flow *flow;
1329                 u32 bucket, obj;
1330
1331                 bucket = cb->args[0];
1332                 obj = cb->args[1];
1333                 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1334                 if (!flow)
1335                         break;
1336
1337                 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1338                                            NETLINK_CB(cb->skb).portid,
1339                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1340                                            OVS_FLOW_CMD_NEW, ufid_flags) < 0)
1341                         break;
1342
1343                 cb->args[0] = bucket;
1344                 cb->args[1] = obj;
1345         }
1346         rcu_read_unlock();
1347         return skb->len;
1348 }
1349
1350 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1351         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1352         [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1353         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1354         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1355         [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1356         [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1357         [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1358 };
1359
1360 static struct genl_ops dp_flow_genl_ops[] = {
1361         { .cmd = OVS_FLOW_CMD_NEW,
1362           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1363           .policy = flow_policy,
1364           .doit = ovs_flow_cmd_new
1365         },
1366         { .cmd = OVS_FLOW_CMD_DEL,
1367           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1368           .policy = flow_policy,
1369           .doit = ovs_flow_cmd_del
1370         },
1371         { .cmd = OVS_FLOW_CMD_GET,
1372           .flags = 0,               /* OK for unprivileged users. */
1373           .policy = flow_policy,
1374           .doit = ovs_flow_cmd_get,
1375           .dumpit = ovs_flow_cmd_dump
1376         },
1377         { .cmd = OVS_FLOW_CMD_SET,
1378           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1379           .policy = flow_policy,
1380           .doit = ovs_flow_cmd_set,
1381         },
1382 };
1383
1384 static struct genl_family dp_flow_genl_family = {
1385         .id = GENL_ID_GENERATE,
1386         .hdrsize = sizeof(struct ovs_header),
1387         .name = OVS_FLOW_FAMILY,
1388         .version = OVS_FLOW_VERSION,
1389         .maxattr = OVS_FLOW_ATTR_MAX,
1390         .netnsok = true,
1391         .parallel_ops = true,
1392         .ops = dp_flow_genl_ops,
1393         .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1394         .mcgrps = &ovs_dp_flow_multicast_group,
1395         .n_mcgrps = 1,
1396 };
1397
1398 static size_t ovs_dp_cmd_msg_size(void)
1399 {
1400         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1401
1402         msgsize += nla_total_size(IFNAMSIZ);
1403         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1404         msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1405         msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1406
1407         return msgsize;
1408 }
1409
1410 /* Called with ovs_mutex. */
1411 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1412                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1413 {
1414         struct ovs_header *ovs_header;
1415         struct ovs_dp_stats dp_stats;
1416         struct ovs_dp_megaflow_stats dp_megaflow_stats;
1417         int err;
1418
1419         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1420                                    flags, cmd);
1421         if (!ovs_header)
1422                 goto error;
1423
1424         ovs_header->dp_ifindex = get_dpifindex(dp);
1425
1426         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1427         if (err)
1428                 goto nla_put_failure;
1429
1430         get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1431         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1432                         &dp_stats))
1433                 goto nla_put_failure;
1434
1435         if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1436                         sizeof(struct ovs_dp_megaflow_stats),
1437                         &dp_megaflow_stats))
1438                 goto nla_put_failure;
1439
1440         if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1441                 goto nla_put_failure;
1442
1443         genlmsg_end(skb, ovs_header);
1444         return 0;
1445
1446 nla_put_failure:
1447         genlmsg_cancel(skb, ovs_header);
1448 error:
1449         return -EMSGSIZE;
1450 }
1451
1452 static struct sk_buff *ovs_dp_cmd_alloc_info(struct genl_info *info)
1453 {
1454         return genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1455 }
1456
1457 /* Called with rcu_read_lock or ovs_mutex. */
1458 static struct datapath *lookup_datapath(struct net *net,
1459                                         const struct ovs_header *ovs_header,
1460                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1461 {
1462         struct datapath *dp;
1463
1464         if (!a[OVS_DP_ATTR_NAME])
1465                 dp = get_dp(net, ovs_header->dp_ifindex);
1466         else {
1467                 struct vport *vport;
1468
1469                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1470                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1471         }
1472         return dp ? dp : ERR_PTR(-ENODEV);
1473 }
1474
1475 static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1476 {
1477         struct datapath *dp;
1478
1479         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1480         if (IS_ERR(dp))
1481                 return;
1482
1483         WARN(dp->user_features, "Dropping previously announced user features\n");
1484         dp->user_features = 0;
1485 }
1486
1487 static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1488 {
1489         if (a[OVS_DP_ATTR_USER_FEATURES])
1490                 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1491 }
1492
1493 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1494 {
1495         struct nlattr **a = info->attrs;
1496         struct vport_parms parms;
1497         struct sk_buff *reply;
1498         struct datapath *dp;
1499         struct vport *vport;
1500         struct ovs_net *ovs_net;
1501         int err, i;
1502
1503         err = -EINVAL;
1504         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1505                 goto err;
1506
1507         reply = ovs_dp_cmd_alloc_info(info);
1508         if (!reply)
1509                 return -ENOMEM;
1510
1511         err = -ENOMEM;
1512         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1513         if (dp == NULL)
1514                 goto err_free_reply;
1515
1516         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1517
1518         /* Allocate table. */
1519         err = ovs_flow_tbl_init(&dp->table);
1520         if (err)
1521                 goto err_free_dp;
1522
1523         dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1524         if (!dp->stats_percpu) {
1525                 err = -ENOMEM;
1526                 goto err_destroy_table;
1527         }
1528
1529         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1530                             GFP_KERNEL);
1531         if (!dp->ports) {
1532                 err = -ENOMEM;
1533                 goto err_destroy_percpu;
1534         }
1535
1536         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1537                 INIT_HLIST_HEAD(&dp->ports[i]);
1538
1539         /* Set up our datapath device. */
1540         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1541         parms.type = OVS_VPORT_TYPE_INTERNAL;
1542         parms.options = NULL;
1543         parms.dp = dp;
1544         parms.port_no = OVSP_LOCAL;
1545         parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1546
1547         ovs_dp_change(dp, a);
1548
1549         /* So far only local changes have been made, now need the lock. */
1550         ovs_lock();
1551
1552         vport = new_vport(&parms);
1553         if (IS_ERR(vport)) {
1554                 err = PTR_ERR(vport);
1555                 if (err == -EBUSY)
1556                         err = -EEXIST;
1557
1558                 if (err == -EEXIST) {
1559                         /* An outdated user space instance that does not understand
1560                          * the concept of user_features has attempted to create a new
1561                          * datapath and is likely to reuse it. Drop all user features.
1562                          */
1563                         if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1564                                 ovs_dp_reset_user_features(skb, info);
1565                 }
1566
1567                 goto err_destroy_ports_array;
1568         }
1569
1570         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1571                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1572         BUG_ON(err < 0);
1573
1574         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1575         list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1576         ovs_unlock();
1577
1578         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1579         return 0;
1580
1581 err_destroy_ports_array:
1582         ovs_unlock();
1583         kfree(dp->ports);
1584 err_destroy_percpu:
1585         free_percpu(dp->stats_percpu);
1586 err_destroy_table:
1587         ovs_flow_tbl_destroy(&dp->table);
1588 err_free_dp:
1589         release_net(ovs_dp_get_net(dp));
1590         kfree(dp);
1591 err_free_reply:
1592         kfree_skb(reply);
1593 err:
1594         return err;
1595 }
1596
1597 /* Called with ovs_mutex. */
1598 static void __dp_destroy(struct datapath *dp)
1599 {
1600         int i;
1601
1602         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1603                 struct vport *vport;
1604                 struct hlist_node *n;
1605
1606                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1607                         if (vport->port_no != OVSP_LOCAL)
1608                                 ovs_dp_detach_port(vport);
1609         }
1610
1611         list_del_rcu(&dp->list_node);
1612
1613         /* OVSP_LOCAL is datapath internal port. We need to make sure that
1614          * all ports in datapath are destroyed first before freeing datapath.
1615          */
1616         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1617
1618         /* RCU destroy the flow table */
1619         call_rcu(&dp->rcu, destroy_dp_rcu);
1620 }
1621
1622 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1623 {
1624         struct sk_buff *reply;
1625         struct datapath *dp;
1626         int err;
1627
1628         reply = ovs_dp_cmd_alloc_info(info);
1629         if (!reply)
1630                 return -ENOMEM;
1631
1632         ovs_lock();
1633         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1634         err = PTR_ERR(dp);
1635         if (IS_ERR(dp))
1636                 goto err_unlock_free;
1637
1638         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1639                                    info->snd_seq, 0, OVS_DP_CMD_DEL);
1640         BUG_ON(err < 0);
1641
1642         __dp_destroy(dp);
1643         ovs_unlock();
1644
1645         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1646         return 0;
1647
1648 err_unlock_free:
1649         ovs_unlock();
1650         kfree_skb(reply);
1651         return err;
1652 }
1653
1654 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1655 {
1656         struct sk_buff *reply;
1657         struct datapath *dp;
1658         int err;
1659
1660         reply = ovs_dp_cmd_alloc_info(info);
1661         if (!reply)
1662                 return -ENOMEM;
1663
1664         ovs_lock();
1665         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1666         err = PTR_ERR(dp);
1667         if (IS_ERR(dp))
1668                 goto err_unlock_free;
1669
1670         ovs_dp_change(dp, info->attrs);
1671
1672         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1673                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1674         BUG_ON(err < 0);
1675         ovs_unlock();
1676
1677         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1678         return 0;
1679
1680 err_unlock_free:
1681         ovs_unlock();
1682         kfree_skb(reply);
1683         return err;
1684 }
1685
1686 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1687 {
1688         struct sk_buff *reply;
1689         struct datapath *dp;
1690         int err;
1691
1692         reply = ovs_dp_cmd_alloc_info(info);
1693         if (!reply)
1694                 return -ENOMEM;
1695
1696         ovs_lock();
1697         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1698         if (IS_ERR(dp)) {
1699                 err = PTR_ERR(dp);
1700                 goto err_unlock_free;
1701         }
1702         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1703                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1704         BUG_ON(err < 0);
1705         ovs_unlock();
1706
1707         return genlmsg_reply(reply, info);
1708
1709 err_unlock_free:
1710         ovs_unlock();
1711         kfree_skb(reply);
1712         return err;
1713 }
1714
1715 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1716 {
1717         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1718         struct datapath *dp;
1719         int skip = cb->args[0];
1720         int i = 0;
1721
1722         ovs_lock();
1723         list_for_each_entry(dp, &ovs_net->dps, list_node) {
1724                 if (i >= skip &&
1725                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1726                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1727                                          OVS_DP_CMD_NEW) < 0)
1728                         break;
1729                 i++;
1730         }
1731         ovs_unlock();
1732
1733         cb->args[0] = i;
1734
1735         return skb->len;
1736 }
1737
1738 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1739         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1740         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1741         [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1742 };
1743
1744 static struct genl_ops dp_datapath_genl_ops[] = {
1745         { .cmd = OVS_DP_CMD_NEW,
1746           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1747           .policy = datapath_policy,
1748           .doit = ovs_dp_cmd_new
1749         },
1750         { .cmd = OVS_DP_CMD_DEL,
1751           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1752           .policy = datapath_policy,
1753           .doit = ovs_dp_cmd_del
1754         },
1755         { .cmd = OVS_DP_CMD_GET,
1756           .flags = 0,               /* OK for unprivileged users. */
1757           .policy = datapath_policy,
1758           .doit = ovs_dp_cmd_get,
1759           .dumpit = ovs_dp_cmd_dump
1760         },
1761         { .cmd = OVS_DP_CMD_SET,
1762           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1763           .policy = datapath_policy,
1764           .doit = ovs_dp_cmd_set,
1765         },
1766 };
1767
1768 static struct genl_family dp_datapath_genl_family = {
1769         .id = GENL_ID_GENERATE,
1770         .hdrsize = sizeof(struct ovs_header),
1771         .name = OVS_DATAPATH_FAMILY,
1772         .version = OVS_DATAPATH_VERSION,
1773         .maxattr = OVS_DP_ATTR_MAX,
1774         .netnsok = true,
1775         .parallel_ops = true,
1776         .ops = dp_datapath_genl_ops,
1777         .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1778         .mcgrps = &ovs_dp_datapath_multicast_group,
1779         .n_mcgrps = 1,
1780 };
1781
1782 /* Called with ovs_mutex or RCU read lock. */
1783 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1784                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1785 {
1786         struct ovs_header *ovs_header;
1787         struct ovs_vport_stats vport_stats;
1788         int err;
1789
1790         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1791                                  flags, cmd);
1792         if (!ovs_header)
1793                 return -EMSGSIZE;
1794
1795         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1796
1797         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1798             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1799             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)))
1800                 goto nla_put_failure;
1801
1802         ovs_vport_get_stats(vport, &vport_stats);
1803         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1804                     &vport_stats))
1805                 goto nla_put_failure;
1806
1807         if (ovs_vport_get_upcall_portids(vport, skb))
1808                 goto nla_put_failure;
1809
1810         err = ovs_vport_get_options(vport, skb);
1811         if (err == -EMSGSIZE)
1812                 goto error;
1813
1814         genlmsg_end(skb, ovs_header);
1815         return 0;
1816
1817 nla_put_failure:
1818         err = -EMSGSIZE;
1819 error:
1820         genlmsg_cancel(skb, ovs_header);
1821         return err;
1822 }
1823
1824 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1825 {
1826         return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1827 }
1828
1829 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1830 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1831                                          u32 seq, u8 cmd)
1832 {
1833         struct sk_buff *skb;
1834         int retval;
1835
1836         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1837         if (!skb)
1838                 return ERR_PTR(-ENOMEM);
1839
1840         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1841         BUG_ON(retval < 0);
1842
1843         return skb;
1844 }
1845
1846 /* Called with ovs_mutex or RCU read lock. */
1847 static struct vport *lookup_vport(struct net *net,
1848                                   const struct ovs_header *ovs_header,
1849                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1850 {
1851         struct datapath *dp;
1852         struct vport *vport;
1853
1854         if (a[OVS_VPORT_ATTR_NAME]) {
1855                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1856                 if (!vport)
1857                         return ERR_PTR(-ENODEV);
1858                 if (ovs_header->dp_ifindex &&
1859                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1860                         return ERR_PTR(-ENODEV);
1861                 return vport;
1862         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1863                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1864
1865                 if (port_no >= DP_MAX_PORTS)
1866                         return ERR_PTR(-EFBIG);
1867
1868                 dp = get_dp(net, ovs_header->dp_ifindex);
1869                 if (!dp)
1870                         return ERR_PTR(-ENODEV);
1871
1872                 vport = ovs_vport_ovsl_rcu(dp, port_no);
1873                 if (!vport)
1874                         return ERR_PTR(-ENODEV);
1875                 return vport;
1876         } else
1877                 return ERR_PTR(-EINVAL);
1878 }
1879
1880 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1881 {
1882         struct nlattr **a = info->attrs;
1883         struct ovs_header *ovs_header = info->userhdr;
1884         struct vport_parms parms;
1885         struct sk_buff *reply;
1886         struct vport *vport;
1887         struct datapath *dp;
1888         u32 port_no;
1889         int err;
1890
1891         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1892             !a[OVS_VPORT_ATTR_UPCALL_PID])
1893                 return -EINVAL;
1894
1895         port_no = a[OVS_VPORT_ATTR_PORT_NO]
1896                 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
1897         if (port_no >= DP_MAX_PORTS)
1898                 return -EFBIG;
1899
1900         reply = ovs_vport_cmd_alloc_info();
1901         if (!reply)
1902                 return -ENOMEM;
1903
1904         ovs_lock();
1905 restart:
1906         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1907         err = -ENODEV;
1908         if (!dp)
1909                 goto exit_unlock_free;
1910
1911         if (port_no) {
1912                 vport = ovs_vport_ovsl(dp, port_no);
1913                 err = -EBUSY;
1914                 if (vport)
1915                         goto exit_unlock_free;
1916         } else {
1917                 for (port_no = 1; ; port_no++) {
1918                         if (port_no >= DP_MAX_PORTS) {
1919                                 err = -EFBIG;
1920                                 goto exit_unlock_free;
1921                         }
1922                         vport = ovs_vport_ovsl(dp, port_no);
1923                         if (!vport)
1924                                 break;
1925                 }
1926         }
1927
1928         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1929         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1930         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1931         parms.dp = dp;
1932         parms.port_no = port_no;
1933         parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
1934
1935         vport = new_vport(&parms);
1936         err = PTR_ERR(vport);
1937         if (IS_ERR(vport)) {
1938                 if (err == -EAGAIN)
1939                         goto restart;
1940                 goto exit_unlock_free;
1941         }
1942
1943         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1944                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1945         BUG_ON(err < 0);
1946         ovs_unlock();
1947
1948         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1949         return 0;
1950
1951 exit_unlock_free:
1952         ovs_unlock();
1953         kfree_skb(reply);
1954         return err;
1955 }
1956
1957 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
1958 {
1959         struct nlattr **a = info->attrs;
1960         struct sk_buff *reply;
1961         struct vport *vport;
1962         int err;
1963
1964         reply = ovs_vport_cmd_alloc_info();
1965         if (!reply)
1966                 return -ENOMEM;
1967
1968         ovs_lock();
1969         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1970         err = PTR_ERR(vport);
1971         if (IS_ERR(vport))
1972                 goto exit_unlock_free;
1973
1974         if (a[OVS_VPORT_ATTR_TYPE] &&
1975             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1976                 err = -EINVAL;
1977                 goto exit_unlock_free;
1978         }
1979
1980         if (a[OVS_VPORT_ATTR_OPTIONS]) {
1981                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
1982                 if (err)
1983                         goto exit_unlock_free;
1984         }
1985
1986         if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
1987                 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
1988
1989                 err = ovs_vport_set_upcall_portids(vport, ids);
1990                 if (err)
1991                         goto exit_unlock_free;
1992         }
1993
1994         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1995                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1996         BUG_ON(err < 0);
1997         ovs_unlock();
1998
1999         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
2000         return 0;
2001
2002 exit_unlock_free:
2003         ovs_unlock();
2004         kfree_skb(reply);
2005         return err;
2006 }
2007
2008 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2009 {
2010         struct nlattr **a = info->attrs;
2011         struct sk_buff *reply;
2012         struct vport *vport;
2013         int err;
2014
2015         reply = ovs_vport_cmd_alloc_info();
2016         if (!reply)
2017                 return -ENOMEM;
2018
2019         ovs_lock();
2020         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2021         err = PTR_ERR(vport);
2022         if (IS_ERR(vport))
2023                 goto exit_unlock_free;
2024
2025         if (vport->port_no == OVSP_LOCAL) {
2026                 err = -EINVAL;
2027                 goto exit_unlock_free;
2028         }
2029
2030         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2031                                       info->snd_seq, 0, OVS_VPORT_CMD_DEL);
2032         BUG_ON(err < 0);
2033         ovs_dp_detach_port(vport);
2034         ovs_unlock();
2035
2036         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
2037         return 0;
2038
2039 exit_unlock_free:
2040         ovs_unlock();
2041         kfree_skb(reply);
2042         return err;
2043 }
2044
2045 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2046 {
2047         struct nlattr **a = info->attrs;
2048         struct ovs_header *ovs_header = info->userhdr;
2049         struct sk_buff *reply;
2050         struct vport *vport;
2051         int err;
2052
2053         reply = ovs_vport_cmd_alloc_info();
2054         if (!reply)
2055                 return -ENOMEM;
2056
2057         rcu_read_lock();
2058         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2059         err = PTR_ERR(vport);
2060         if (IS_ERR(vport))
2061                 goto exit_unlock_free;
2062         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2063                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2064         BUG_ON(err < 0);
2065         rcu_read_unlock();
2066
2067         return genlmsg_reply(reply, info);
2068
2069 exit_unlock_free:
2070         rcu_read_unlock();
2071         kfree_skb(reply);
2072         return err;
2073 }
2074
2075 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2076 {
2077         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2078         struct datapath *dp;
2079         int bucket = cb->args[0], skip = cb->args[1];
2080         int i, j = 0;
2081
2082         rcu_read_lock();
2083         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2084         if (!dp) {
2085                 rcu_read_unlock();
2086                 return -ENODEV;
2087         }
2088         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2089                 struct vport *vport;
2090
2091                 j = 0;
2092                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2093                         if (j >= skip &&
2094                             ovs_vport_cmd_fill_info(vport, skb,
2095                                                     NETLINK_CB(cb->skb).portid,
2096                                                     cb->nlh->nlmsg_seq,
2097                                                     NLM_F_MULTI,
2098                                                     OVS_VPORT_CMD_NEW) < 0)
2099                                 goto out;
2100
2101                         j++;
2102                 }
2103                 skip = 0;
2104         }
2105 out:
2106         rcu_read_unlock();
2107
2108         cb->args[0] = i;
2109         cb->args[1] = j;
2110
2111         return skb->len;
2112 }
2113
2114 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2115         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2116         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2117         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2118         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2119         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2120         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2121 };
2122
2123 static struct genl_ops dp_vport_genl_ops[] = {
2124         { .cmd = OVS_VPORT_CMD_NEW,
2125           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2126           .policy = vport_policy,
2127           .doit = ovs_vport_cmd_new
2128         },
2129         { .cmd = OVS_VPORT_CMD_DEL,
2130           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2131           .policy = vport_policy,
2132           .doit = ovs_vport_cmd_del
2133         },
2134         { .cmd = OVS_VPORT_CMD_GET,
2135           .flags = 0,               /* OK for unprivileged users. */
2136           .policy = vport_policy,
2137           .doit = ovs_vport_cmd_get,
2138           .dumpit = ovs_vport_cmd_dump
2139         },
2140         { .cmd = OVS_VPORT_CMD_SET,
2141           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2142           .policy = vport_policy,
2143           .doit = ovs_vport_cmd_set,
2144         },
2145 };
2146
2147 struct genl_family dp_vport_genl_family = {
2148         .id = GENL_ID_GENERATE,
2149         .hdrsize = sizeof(struct ovs_header),
2150         .name = OVS_VPORT_FAMILY,
2151         .version = OVS_VPORT_VERSION,
2152         .maxattr = OVS_VPORT_ATTR_MAX,
2153         .netnsok = true,
2154         .parallel_ops = true,
2155         .ops = dp_vport_genl_ops,
2156         .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2157         .mcgrps = &ovs_dp_vport_multicast_group,
2158         .n_mcgrps = 1,
2159 };
2160
2161 static struct genl_family *dp_genl_families[] = {
2162         &dp_datapath_genl_family,
2163         &dp_vport_genl_family,
2164         &dp_flow_genl_family,
2165         &dp_packet_genl_family,
2166 };
2167
2168 static void dp_unregister_genl(int n_families)
2169 {
2170         int i;
2171
2172         for (i = 0; i < n_families; i++)
2173                 genl_unregister_family(dp_genl_families[i]);
2174 }
2175
2176 static int dp_register_genl(void)
2177 {
2178         int err;
2179         int i;
2180
2181         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2182
2183                 err = genl_register_family(dp_genl_families[i]);
2184                 if (err)
2185                         goto error;
2186         }
2187
2188         return 0;
2189
2190 error:
2191         dp_unregister_genl(i);
2192         return err;
2193 }
2194
2195 static int __net_init ovs_init_net(struct net *net)
2196 {
2197         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2198
2199         INIT_LIST_HEAD(&ovs_net->dps);
2200         INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2201         return 0;
2202 }
2203
2204 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2205                                             struct list_head *head)
2206 {
2207         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2208         struct datapath *dp;
2209
2210         list_for_each_entry(dp, &ovs_net->dps, list_node) {
2211                 int i;
2212
2213                 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2214                         struct vport *vport;
2215
2216                         hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2217                                 struct netdev_vport *netdev_vport;
2218
2219                                 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2220                                         continue;
2221
2222                                 netdev_vport = netdev_vport_priv(vport);
2223                                 if (dev_net(netdev_vport->dev) == dnet)
2224                                         list_add(&vport->detach_list, head);
2225                         }
2226                 }
2227         }
2228 }
2229
2230 static void __net_exit ovs_exit_net(struct net *dnet)
2231 {
2232         struct datapath *dp, *dp_next;
2233         struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2234         struct vport *vport, *vport_next;
2235         struct net *net;
2236         LIST_HEAD(head);
2237
2238         ovs_lock();
2239         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2240                 __dp_destroy(dp);
2241
2242         rtnl_lock();
2243         for_each_net(net)
2244                 list_vports_from_net(net, dnet, &head);
2245         rtnl_unlock();
2246
2247         /* Detach all vports from given namespace. */
2248         list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2249                 list_del(&vport->detach_list);
2250                 ovs_dp_detach_port(vport);
2251         }
2252
2253         ovs_unlock();
2254
2255         cancel_work_sync(&ovs_net->dp_notify_work);
2256 }
2257
2258 static struct pernet_operations ovs_net_ops = {
2259         .init = ovs_init_net,
2260         .exit = ovs_exit_net,
2261         .id   = &ovs_net_id,
2262         .size = sizeof(struct ovs_net),
2263 };
2264
2265 DEFINE_COMPAT_PNET_REG_FUNC(device);
2266
2267 static int __init dp_init(void)
2268 {
2269         int err;
2270
2271         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2272
2273         pr_info("Open vSwitch switching datapath %s\n", VERSION);
2274
2275         err = action_fifos_init();
2276         if (err)
2277                 goto error;
2278
2279         err = ovs_internal_dev_rtnl_link_register();
2280         if (err)
2281                 goto error_action_fifos_exit;
2282
2283         err = ovs_flow_init();
2284         if (err)
2285                 goto error_unreg_rtnl_link;
2286
2287         err = ovs_vport_init();
2288         if (err)
2289                 goto error_flow_exit;
2290
2291         err = register_pernet_device(&ovs_net_ops);
2292         if (err)
2293                 goto error_vport_exit;
2294
2295         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2296         if (err)
2297                 goto error_netns_exit;
2298
2299         err = ovs_netdev_init();
2300         if (err)
2301                 goto error_unreg_notifier;
2302
2303         err = dp_register_genl();
2304         if (err < 0)
2305                 goto error_unreg_netdev;
2306
2307         return 0;
2308
2309 error_unreg_netdev:
2310         ovs_netdev_exit();
2311 error_unreg_notifier:
2312         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2313 error_netns_exit:
2314         unregister_pernet_device(&ovs_net_ops);
2315 error_vport_exit:
2316         ovs_vport_exit();
2317 error_flow_exit:
2318         ovs_flow_exit();
2319 error_unreg_rtnl_link:
2320         ovs_internal_dev_rtnl_link_unregister();
2321 error_action_fifos_exit:
2322         action_fifos_exit();
2323 error:
2324         return err;
2325 }
2326
2327 static void dp_cleanup(void)
2328 {
2329         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2330         ovs_netdev_exit();
2331         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2332         unregister_pernet_device(&ovs_net_ops);
2333         rcu_barrier();
2334         ovs_vport_exit();
2335         ovs_flow_exit();
2336         ovs_internal_dev_rtnl_link_unregister();
2337         action_fifos_exit();
2338 }
2339
2340 module_init(dp_init);
2341 module_exit(dp_cleanup);
2342
2343 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2344 MODULE_LICENSE("GPL");
2345 MODULE_VERSION(VERSION);