datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2015 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <net/genetlink.h>
  52 #include <net/net_namespace.h>
  53 #include <net/netns/generic.h>
  54
  55 #include "datapath.h"
  56 #include "flow.h"
  57 #include "flow_table.h"
  58 #include "flow_netlink.h"
  59 #include "gso.h"
  60 #include "vlan.h"
  61 #include "vport-internal_dev.h"
  62 #include "vport-netdev.h"
  63
  64 int ovs_net_id __read_mostly;
  65 EXPORT_SYMBOL_GPL(ovs_net_id);
  66
  67 static struct genl_family dp_packet_genl_family;
  68 static struct genl_family dp_flow_genl_family;
  69 static struct genl_family dp_datapath_genl_family;
  70
  71 static const struct nla_policy flow_policy[];
  72
  73 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
  74         .name = OVS_FLOW_MCGROUP
  75 };
  76
  77 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  78         .name = OVS_DATAPATH_MCGROUP
  79 };
  80
  81 struct genl_multicast_group ovs_dp_vport_multicast_group = {
  82         .name = OVS_VPORT_MCGROUP
  83 };
  84
  85 /* Check if need to build a reply message.
  86  * OVS userspace sets the NLM_F_ECHO flag if it needs the reply.
  87  */
  88 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
  89                             unsigned int group)
  90 {
  91         return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  92                genl_has_listeners(family, genl_info_net(info), group);
  93 }
  94
  95 static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
  96                        struct sk_buff *skb, struct genl_info *info)
  97 {
  98         genl_notify(family, skb, genl_info_net(info),
  99                     info->snd_portid, GROUP_ID(grp), info->nlhdr, GFP_KERNEL);
 100 }
 101
 102 /**
 103  * DOC: Locking:
 104  *
 105  * All writes e.g. Writes to device state (add/remove datapath, port, set
 106  * operations on vports, etc.), Writes to other state (flow table
 107  * modifications, set miscellaneous datapath parameters, etc.) are protected
 108  * by ovs_lock.
 109  *
 110  * Reads are protected by RCU.
 111  *
 112  * There are a few special cases (mostly stats) that have their own
 113  * synchronization but they nest under all of above and don't interact with
 114  * each other.
 115  *
 116  * The RTNL lock nests inside ovs_mutex.
 117  */
 118
 119 static DEFINE_MUTEX(ovs_mutex);
 120
 121 void ovs_lock(void)
 122 {
 123         mutex_lock(&ovs_mutex);
 124 }
 125
 126 void ovs_unlock(void)
 127 {
 128         mutex_unlock(&ovs_mutex);
 129 }
 130
 131 #ifdef CONFIG_LOCKDEP
 132 int lockdep_ovsl_is_held(void)
 133 {
 134         if (debug_locks)
 135                 return lockdep_is_held(&ovs_mutex);
 136         else
 137                 return 1;
 138 }
 139 EXPORT_SYMBOL_GPL(lockdep_ovsl_is_held);
 140 #endif
 141
 142 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 143                              const struct sw_flow_key *,
 144                              const struct dp_upcall_info *);
 145 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 146                                   const struct sw_flow_key *,
 147                                   const struct dp_upcall_info *);
 148
 149 /* Must be called with rcu_read_lock. */
 150 static struct datapath *get_dp_rcu(struct net *net, int dp_ifindex)
 151 {
 152         struct net_device *dev = dev_get_by_index_rcu(net, dp_ifindex);
 153
 154         if (dev) {
 155                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 156                 if (vport)
 157                         return vport->dp;
 158         }
 159
 160         return NULL;
 161 }
 162
 163 /* The caller must hold either ovs_mutex or rcu_read_lock to keep the
 164  * returned dp pointer valid.
 165  */
 166 static inline struct datapath *get_dp(struct net *net, int dp_ifindex)
 167 {
 168         struct datapath *dp;
 169
 170         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_ovsl_is_held());
 171         rcu_read_lock();
 172         dp = get_dp_rcu(net, dp_ifindex);
 173         rcu_read_unlock();
 174
 175         return dp;
 176 }
 177
 178 /* Must be called with rcu_read_lock or ovs_mutex. */
 179 const char *ovs_dp_name(const struct datapath *dp)
 180 {
 181         struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 182         return ovs_vport_name(vport);
 183 }
 184
 185 static int get_dpifindex(const struct datapath *dp)
 186 {
 187         struct vport *local;
 188         int ifindex;
 189
 190         rcu_read_lock();
 191
 192         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 193         if (local)
 194                 ifindex = local->dev->ifindex;
 195         else
 196                 ifindex = 0;
 197
 198         rcu_read_unlock();
 199
 200         return ifindex;
 201 }
 202
 203 static void destroy_dp_rcu(struct rcu_head *rcu)
 204 {
 205         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 206
 207         ovs_flow_tbl_destroy(&dp->table);
 208         free_percpu(dp->stats_percpu);
 209         kfree(dp->ports);
 210         kfree(dp);
 211 }
 212
 213 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 214                                             u16 port_no)
 215 {
 216         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 217 }
 218
 219 /* Called with ovs_mutex or RCU read lock. */
 220 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 221 {
 222         struct vport *vport;
 223         struct hlist_head *head;
 224
 225         head = vport_hash_bucket(dp, port_no);
 226         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 227                 if (vport->port_no == port_no)
 228                         return vport;
 229         }
 230         return NULL;
 231 }
 232
 233 /* Called with ovs_mutex. */
 234 static struct vport *new_vport(const struct vport_parms *parms)
 235 {
 236         struct vport *vport;
 237
 238         vport = ovs_vport_add(parms);
 239         if (!IS_ERR(vport)) {
 240                 struct datapath *dp = parms->dp;
 241                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 242
 243                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 244         }
 245         return vport;
 246 }
 247
 248 void ovs_dp_detach_port(struct vport *p)
 249 {
 250         ASSERT_OVSL();
 251
 252         /* First drop references to device. */
 253         hlist_del_rcu(&p->dp_hash_node);
 254
 255         /* Then destroy it. */
 256         ovs_vport_del(p);
 257 }
 258
 259 /* Must be called with rcu_read_lock. */
 260 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
 261 {
 262         const struct vport *p = OVS_CB(skb)->input_vport;
 263         struct datapath *dp = p->dp;
 264         struct sw_flow *flow;
 265         struct sw_flow_actions *sf_acts;
 266         struct dp_stats_percpu *stats;
 267         u64 *stats_counter;
 268         u32 n_mask_hit;
 269
 270         stats = this_cpu_ptr(dp->stats_percpu);
 271
 272         /* Look up flow. */
 273         flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
 274                                          &n_mask_hit);
 275         if (unlikely(!flow)) {
 276                 struct dp_upcall_info upcall;
 277                 int error;
 278
 279                 memset(&upcall, 0, sizeof(upcall));
 280                 upcall.cmd = OVS_PACKET_CMD_MISS;
 281                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 282                 error = ovs_dp_upcall(dp, skb, key, &upcall);
 283                 if (unlikely(error))
 284                         kfree_skb(skb);
 285                 else
 286                         consume_skb(skb);
 287                 stats_counter = &stats->n_missed;
 288                 goto out;
 289         }
 290
 291         ovs_flow_stats_update(flow, key->tp.flags, skb);
 292         sf_acts = rcu_dereference(flow->sf_acts);
 293         ovs_execute_actions(dp, skb, sf_acts, key);
 294
 295         stats_counter = &stats->n_hit;
 296
 297 out:
 298         /* Update datapath statistics. */
 299         u64_stats_update_begin(&stats->syncp);
 300         (*stats_counter)++;
 301         stats->n_mask_hit += n_mask_hit;
 302         u64_stats_update_end(&stats->syncp);
 303 }
 304
 305 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 306                   const struct sw_flow_key *key,
 307                   const struct dp_upcall_info *upcall_info)
 308 {
 309         struct dp_stats_percpu *stats;
 310         int err;
 311
 312         if (upcall_info->portid == 0) {
 313                 err = -ENOTCONN;
 314                 goto err;
 315         }
 316
 317         if (!skb_is_gso(skb))
 318                 err = queue_userspace_packet(dp, skb, key, upcall_info);
 319         else
 320                 err = queue_gso_packets(dp, skb, key, upcall_info);
 321         if (err)
 322                 goto err;
 323
 324         return 0;
 325
 326 err:
 327         stats = this_cpu_ptr(dp->stats_percpu);
 328
 329         u64_stats_update_begin(&stats->syncp);
 330         stats->n_lost++;
 331         u64_stats_update_end(&stats->syncp);
 332
 333         return err;
 334 }
 335
 336 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 337                              const struct sw_flow_key *key,
 338                              const struct dp_upcall_info *upcall_info)
 339 {
 340         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 341         struct sw_flow_key later_key;
 342         struct sk_buff *segs, *nskb;
 343         struct ovs_skb_cb ovs_cb;
 344         int err;
 345
 346         ovs_cb = *OVS_CB(skb);
 347         segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 348         *OVS_CB(skb) = ovs_cb;
 349         if (IS_ERR(segs))
 350                 return PTR_ERR(segs);
 351         if (segs == NULL)
 352                 return -EINVAL;
 353
 354         if (gso_type & SKB_GSO_UDP) {
 355                 /* The initial flow key extracted by ovs_flow_key_extract()
 356                  * in this case is for a first fragment, so we need to
 357                  * properly mark later fragments.
 358                  */
 359                 later_key = *key;
 360                 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 361         }
 362
 363         /* Queue all of the segments. */
 364         skb = segs;
 365         do {
 366                 *OVS_CB(skb) = ovs_cb;
 367                 if (gso_type & SKB_GSO_UDP && skb != segs)
 368                         key = &later_key;
 369
 370                 err = queue_userspace_packet(dp, skb, key, upcall_info);
 371                 if (err)
 372                         break;
 373
 374         } while ((skb = skb->next));
 375
 376         /* Free all of the segments. */
 377         skb = segs;
 378         do {
 379                 nskb = skb->next;
 380                 if (err)
 381                         kfree_skb(skb);
 382                 else
 383                         consume_skb(skb);
 384         } while ((skb = nskb));
 385         return err;
 386 }
 387
 388 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 389                               unsigned int hdrlen)
 390 {
 391         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 392                 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 393                 + nla_total_size(ovs_key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 394
 395         /* OVS_PACKET_ATTR_USERDATA */
 396         if (upcall_info->userdata)
 397                 size += NLA_ALIGN(upcall_info->userdata->nla_len);
 398
 399         /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
 400         if (upcall_info->egress_tun_info)
 401                 size += nla_total_size(ovs_tun_key_attr_size());
 402
 403         /* OVS_PACKET_ATTR_ACTIONS */
 404         if (upcall_info->actions_len)
 405                 size += nla_total_size(upcall_info->actions_len);
 406
 407         return size;
 408 }
 409
 410 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 411                                   const struct sw_flow_key *key,
 412                                   const struct dp_upcall_info *upcall_info)
 413 {
 414         struct ovs_header *upcall;
 415         struct sk_buff *nskb = NULL;
 416         struct sk_buff *user_skb = NULL; /* to be queued to userspace */
 417         struct nlattr *nla;
 418         struct genl_info info = {
 419 #ifdef HAVE_GENLMSG_NEW_UNICAST
 420                 .dst_sk = ovs_dp_get_net(dp)->genl_sock,
 421 #endif
 422                 .snd_portid = upcall_info->portid,
 423         };
 424         size_t len;
 425         unsigned int hlen;
 426         int err, dp_ifindex;
 427
 428         dp_ifindex = get_dpifindex(dp);
 429         if (!dp_ifindex)
 430                 return -ENODEV;
 431
 432         if (skb_vlan_tag_present(skb)) {
 433                 nskb = skb_clone(skb, GFP_ATOMIC);
 434                 if (!nskb)
 435                         return -ENOMEM;
 436
 437                 nskb = vlan_insert_tag_set_proto(nskb, nskb->vlan_proto, skb_vlan_tag_get(nskb));
 438                 if (!nskb)
 439                         return -ENOMEM;
 440
 441                 vlan_set_tci(nskb, 0);
 442
 443                 skb = nskb;
 444         }
 445
 446         if (nla_attr_size(skb->len) > USHRT_MAX) {
 447                 err = -EFBIG;
 448                 goto out;
 449         }
 450
 451         /* Complete checksum if needed */
 452         if (skb->ip_summed == CHECKSUM_PARTIAL &&
 453             (err = skb_checksum_help(skb)))
 454                 goto out;
 455
 456         /* Older versions of OVS user space enforce alignment of the last
 457          * Netlink attribute to NLA_ALIGNTO which would require extensive
 458          * padding logic. Only perform zerocopy if padding is not required.
 459          */
 460         if (dp->user_features & OVS_DP_F_UNALIGNED)
 461                 hlen = skb_zerocopy_headlen(skb);
 462         else
 463                 hlen = skb->len;
 464
 465         len = upcall_msg_size(upcall_info, hlen);
 466         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 467         if (!user_skb) {
 468                 err = -ENOMEM;
 469                 goto out;
 470         }
 471
 472         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 473                              0, upcall_info->cmd);
 474         upcall->dp_ifindex = dp_ifindex;
 475
 476         err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
 477         BUG_ON(err);
 478
 479         if (upcall_info->userdata)
 480                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 481                           nla_len(upcall_info->userdata),
 482                           nla_data(upcall_info->userdata));
 483
 484
 485         if (upcall_info->egress_tun_info) {
 486                 nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_EGRESS_TUN_KEY);
 487                 err = ovs_nla_put_egress_tunnel_key(user_skb,
 488                                                     upcall_info->egress_tun_info,
 489                                                     upcall_info->egress_tun_opts);
 490                 BUG_ON(err);
 491                 nla_nest_end(user_skb, nla);
 492         }
 493
 494         if (upcall_info->actions_len) {
 495                 nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_ACTIONS);
 496                 err = ovs_nla_put_actions(upcall_info->actions,
 497                                           upcall_info->actions_len,
 498                                           user_skb);
 499                 if (!err)
 500                         nla_nest_end(user_skb, nla);
 501                 else
 502                         nla_nest_cancel(user_skb, nla);
 503         }
 504
 505         /* Only reserve room for attribute header, packet data is added
 506          * in skb_zerocopy()
 507          */
 508         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 509                 err = -ENOBUFS;
 510                 goto out;
 511         }
 512         nla->nla_len = nla_attr_size(skb->len);
 513
 514         err = skb_zerocopy(user_skb, skb, skb->len, hlen);
 515         if (err)
 516                 goto out;
 517
 518         /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 519         if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 520                 size_t plen = NLA_ALIGN(user_skb->len) - user_skb->len;
 521
 522                 if (plen > 0)
 523                         memset(skb_put(user_skb, plen), 0, plen);
 524         }
 525
 526         ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 527
 528         err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 529         user_skb = NULL;
 530 out:
 531         if (err)
 532                 skb_tx_error(skb);
 533         kfree_skb(user_skb);
 534         kfree_skb(nskb);
 535         return err;
 536 }
 537
 538 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 539 {
 540         struct ovs_header *ovs_header = info->userhdr;
 541         struct nlattr **a = info->attrs;
 542         struct sw_flow_actions *acts;
 543         struct sk_buff *packet;
 544         struct sw_flow *flow;
 545         struct sw_flow_actions *sf_acts;
 546         struct datapath *dp;
 547         struct ethhdr *eth;
 548         struct vport *input_vport;
 549         int len;
 550         int err;
 551         bool log = !a[OVS_PACKET_ATTR_PROBE];
 552
 553         err = -EINVAL;
 554         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 555             !a[OVS_PACKET_ATTR_ACTIONS])
 556                 goto err;
 557
 558         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 559         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 560         err = -ENOMEM;
 561         if (!packet)
 562                 goto err;
 563         skb_reserve(packet, NET_IP_ALIGN);
 564
 565         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 566
 567         skb_reset_mac_header(packet);
 568         eth = eth_hdr(packet);
 569
 570         /* Normally, setting the skb 'protocol' field would be handled by a
 571          * call to eth_type_trans(), but it assumes there's a sending
 572          * device, which we may not have.
 573          */
 574         if (eth_proto_is_802_3(eth->h_proto))
 575                 packet->protocol = eth->h_proto;
 576         else
 577                 packet->protocol = htons(ETH_P_802_2);
 578
 579         /* Build an sw_flow for sending this packet. */
 580         flow = ovs_flow_alloc();
 581         err = PTR_ERR(flow);
 582         if (IS_ERR(flow))
 583                 goto err_kfree_skb;
 584
 585         err = ovs_flow_key_extract_userspace(a[OVS_PACKET_ATTR_KEY], packet,
 586                                              &flow->key, log);
 587         if (err)
 588                 goto err_flow_free;
 589
 590         err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
 591                                    &flow->key, &acts, log);
 592         if (err)
 593                 goto err_flow_free;
 594
 595         rcu_assign_pointer(flow->sf_acts, acts);
 596         packet->priority = flow->key.phy.priority;
 597         packet->mark = flow->key.phy.skb_mark;
 598
 599         rcu_read_lock();
 600         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
 601         err = -ENODEV;
 602         if (!dp)
 603                 goto err_unlock;
 604
 605         input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 606         if (!input_vport)
 607                 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 608
 609         if (!input_vport)
 610                 goto err_unlock;
 611
 612         packet->dev = input_vport->dev;
 613         OVS_CB(packet)->input_vport = input_vport;
 614         sf_acts = rcu_dereference(flow->sf_acts);
 615
 616         local_bh_disable();
 617         err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
 618         local_bh_enable();
 619         rcu_read_unlock();
 620
 621         ovs_flow_free(flow, false);
 622         return err;
 623
 624 err_unlock:
 625         rcu_read_unlock();
 626 err_flow_free:
 627         ovs_flow_free(flow, false);
 628 err_kfree_skb:
 629         kfree_skb(packet);
 630 err:
 631         return err;
 632 }
 633
 634 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 635         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 636         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 637         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 638         [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
 639 };
 640
 641 static struct genl_ops dp_packet_genl_ops[] = {
 642         { .cmd = OVS_PACKET_CMD_EXECUTE,
 643           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 644           .policy = packet_policy,
 645           .doit = ovs_packet_cmd_execute
 646         }
 647 };
 648
 649 static struct genl_family dp_packet_genl_family = {
 650         .id = GENL_ID_GENERATE,
 651         .hdrsize = sizeof(struct ovs_header),
 652         .name = OVS_PACKET_FAMILY,
 653         .version = OVS_PACKET_VERSION,
 654         .maxattr = OVS_PACKET_ATTR_MAX,
 655         .netnsok = true,
 656         .parallel_ops = true,
 657         .ops = dp_packet_genl_ops,
 658         .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
 659 };
 660
 661 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
 662                          struct ovs_dp_megaflow_stats *mega_stats)
 663 {
 664         int i;
 665
 666         memset(mega_stats, 0, sizeof(*mega_stats));
 667
 668         stats->n_flows = ovs_flow_tbl_count(&dp->table);
 669         mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 670
 671         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 672
 673         for_each_possible_cpu(i) {
 674                 const struct dp_stats_percpu *percpu_stats;
 675                 struct dp_stats_percpu local_stats;
 676                 unsigned int start;
 677
 678                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 679
 680                 do {
 681                         start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
 682                         local_stats = *percpu_stats;
 683                 } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
 684
 685                 stats->n_hit += local_stats.n_hit;
 686                 stats->n_missed += local_stats.n_missed;
 687                 stats->n_lost += local_stats.n_lost;
 688                 mega_stats->n_mask_hit += local_stats.n_mask_hit;
 689         }
 690 }
 691
 692 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
 693 {
 694         return ovs_identifier_is_ufid(sfid) &&
 695                !(ufid_flags & OVS_UFID_F_OMIT_KEY);
 696 }
 697
 698 static bool should_fill_mask(uint32_t ufid_flags)
 699 {
 700         return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
 701 }
 702
 703 static bool should_fill_actions(uint32_t ufid_flags)
 704 {
 705         return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
 706 }
 707
 708 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
 709                                     const struct sw_flow_id *sfid,
 710                                     uint32_t ufid_flags)
 711 {
 712         size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
 713
 714         /* OVS_FLOW_ATTR_UFID */
 715         if (sfid && ovs_identifier_is_ufid(sfid))
 716                 len += nla_total_size(sfid->ufid_len);
 717
 718         /* OVS_FLOW_ATTR_KEY */
 719         if (!sfid || should_fill_key(sfid, ufid_flags))
 720                 len += nla_total_size(ovs_key_attr_size());
 721
 722         /* OVS_FLOW_ATTR_MASK */
 723         if (should_fill_mask(ufid_flags))
 724                 len += nla_total_size(ovs_key_attr_size());
 725
 726         /* OVS_FLOW_ATTR_ACTIONS */
 727         if (should_fill_actions(ufid_flags))
 728                 len += nla_total_size(acts->actions_len);
 729
 730         return len
 731                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 732                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 733                 + nla_total_size(8); /* OVS_FLOW_ATTR_USED */
 734 }
 735
 736 /* Called with ovs_mutex or RCU read lock. */
 737 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
 738                                    struct sk_buff *skb)
 739 {
 740         struct ovs_flow_stats stats;
 741         __be16 tcp_flags;
 742         unsigned long used;
 743
 744         ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 745
 746         if (used &&
 747             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
 748                 return -EMSGSIZE;
 749
 750         if (stats.n_packets &&
 751             nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
 752                 return -EMSGSIZE;
 753
 754         if ((u8)ntohs(tcp_flags) &&
 755              nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 756                 return -EMSGSIZE;
 757
 758         return 0;
 759 }
 760
 761 /* Called with ovs_mutex or RCU read lock. */
 762 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
 763                                      struct sk_buff *skb, int skb_orig_len)
 764 {
 765         struct nlattr *start;
 766         int err;
 767
 768         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 769          * this is the first flow to be dumped into 'skb'.  This is unusual for
 770          * Netlink but individual action lists can be longer than
 771          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 772          * The userspace caller can always fetch the actions separately if it
 773          * really wants them.  (Most userspace callers in fact don't care.)
 774          *
 775          * This can only fail for dump operations because the skb is always
 776          * properly sized for single flows.
 777          */
 778         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 779         if (start) {
 780                 const struct sw_flow_actions *sf_acts;
 781
 782                 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 783                 err = ovs_nla_put_actions(sf_acts->actions,
 784                                           sf_acts->actions_len, skb);
 785
 786                 if (!err)
 787                         nla_nest_end(skb, start);
 788                 else {
 789                         if (skb_orig_len)
 790                                 return err;
 791
 792                         nla_nest_cancel(skb, start);
 793                 }
 794         } else if (skb_orig_len) {
 795                 return -EMSGSIZE;
 796         }
 797
 798         return 0;
 799 }
 800
 801 /* Called with ovs_mutex or RCU read lock. */
 802 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
 803                                   struct sk_buff *skb, u32 portid,
 804                                   u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
 805 {
 806         const int skb_orig_len = skb->len;
 807         struct ovs_header *ovs_header;
 808         int err;
 809
 810         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
 811                                  flags, cmd);
 812         if (!ovs_header)
 813                 return -EMSGSIZE;
 814
 815         ovs_header->dp_ifindex = dp_ifindex;
 816
 817         err = ovs_nla_put_identifier(flow, skb);
 818         if (err)
 819                 goto error;
 820
 821         if (should_fill_key(&flow->id, ufid_flags)) {
 822                 err = ovs_nla_put_masked_key(flow, skb);
 823                 if (err)
 824                         goto error;
 825         }
 826
 827         if (should_fill_mask(ufid_flags)) {
 828                 err = ovs_nla_put_mask(flow, skb);
 829                 if (err)
 830                         goto error;
 831         }
 832
 833         err = ovs_flow_cmd_fill_stats(flow, skb);
 834         if (err)
 835                 goto error;
 836
 837         if (should_fill_actions(ufid_flags)) {
 838                 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
 839                 if (err)
 840                         goto error;
 841         }
 842
 843         genlmsg_end(skb, ovs_header);
 844         return 0;
 845
 846 error:
 847         genlmsg_cancel(skb, ovs_header);
 848         return err;
 849 }
 850
 851 /* May not be called with RCU read lock. */
 852 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 853                                                const struct sw_flow_id *sfid,
 854                                                struct genl_info *info,
 855                                                bool always,
 856                                                uint32_t ufid_flags)
 857 {
 858         struct sk_buff *skb;
 859         size_t len;
 860
 861         if (!always && !ovs_must_notify(&dp_flow_genl_family, info,
 862                                         GROUP_ID(&ovs_dp_flow_multicast_group)))
 863                 return NULL;
 864
 865         len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
 866         skb = genlmsg_new_unicast(len, info, GFP_KERNEL);
 867         if (!skb)
 868                 return ERR_PTR(-ENOMEM);
 869
 870         return skb;
 871 }
 872
 873 /* Called with ovs_mutex. */
 874 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
 875                                                int dp_ifindex,
 876                                                struct genl_info *info, u8 cmd,
 877                                                bool always, u32 ufid_flags)
 878 {
 879         struct sk_buff *skb;
 880         int retval;
 881
 882         skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
 883                                       &flow->id, info, always, ufid_flags);
 884         if (IS_ERR_OR_NULL(skb))
 885                 return skb;
 886
 887         retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
 888                                         info->snd_portid, info->snd_seq, 0,
 889                                         cmd, ufid_flags);
 890         BUG_ON(retval < 0);
 891         return skb;
 892 }
 893
 894 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 895 {
 896         struct nlattr **a = info->attrs;
 897         struct ovs_header *ovs_header = info->userhdr;
 898         struct sw_flow *flow = NULL, *new_flow;
 899         struct sw_flow_mask mask;
 900         struct sk_buff *reply;
 901         struct datapath *dp;
 902         struct sw_flow_key key;
 903         struct sw_flow_actions *acts;
 904         struct sw_flow_match match;
 905         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
 906         int error;
 907         bool log = !a[OVS_FLOW_ATTR_PROBE];
 908
 909         /* Must have key and actions. */
 910         error = -EINVAL;
 911         if (!a[OVS_FLOW_ATTR_KEY]) {
 912                 OVS_NLERR(log, "Flow key attr not present in new flow.");
 913                 goto error;
 914         }
 915         if (!a[OVS_FLOW_ATTR_ACTIONS]) {
 916                 OVS_NLERR(log, "Flow actions attr not present in new flow.");
 917                 goto error;
 918         }
 919
 920         /* Most of the time we need to allocate a new flow, do it before
 921          * locking.
 922          */
 923         new_flow = ovs_flow_alloc();
 924         if (IS_ERR(new_flow)) {
 925                 error = PTR_ERR(new_flow);
 926                 goto error;
 927         }
 928
 929         /* Extract key. */
 930         ovs_match_init(&match, &key, &mask);
 931         error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
 932                                   a[OVS_FLOW_ATTR_MASK], log);
 933         if (error)
 934                 goto err_kfree_flow;
 935
 936         ovs_flow_mask_key(&new_flow->key, &key, true, &mask);
 937
 938         /* Extract flow identifier. */
 939         error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
 940                                        &key, log);
 941         if (error)
 942                 goto err_kfree_flow;
 943
 944         /* Validate actions. */
 945         error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &new_flow->key,
 946                                      &acts, log);
 947         if (error) {
 948                 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
 949                 goto err_kfree_flow;
 950         }
 951
 952         reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
 953                                         ufid_flags);
 954         if (IS_ERR(reply)) {
 955                 error = PTR_ERR(reply);
 956                 goto err_kfree_acts;
 957         }
 958
 959         ovs_lock();
 960         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 961         if (unlikely(!dp)) {
 962                 error = -ENODEV;
 963                 goto err_unlock_ovs;
 964         }
 965
 966         /* Check if this is a duplicate flow */
 967         if (ovs_identifier_is_ufid(&new_flow->id))
 968                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
 969         if (!flow)
 970                 flow = ovs_flow_tbl_lookup(&dp->table, &key);
 971         if (likely(!flow)) {
 972                 rcu_assign_pointer(new_flow->sf_acts, acts);
 973
 974                 /* Put flow in bucket. */
 975                 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
 976                 if (unlikely(error)) {
 977                         acts = NULL;
 978                         goto err_unlock_ovs;
 979                 }
 980
 981                 if (unlikely(reply)) {
 982                         error = ovs_flow_cmd_fill_info(new_flow,
 983                                                        ovs_header->dp_ifindex,
 984                                                        reply, info->snd_portid,
 985                                                        info->snd_seq, 0,
 986                                                        OVS_FLOW_CMD_NEW,
 987                                                        ufid_flags);
 988                         BUG_ON(error < 0);
 989                 }
 990                 ovs_unlock();
 991         } else {
 992                 struct sw_flow_actions *old_acts;
 993
 994                 /* Bail out if we're not allowed to modify an existing flow.
 995                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 996                  * because Generic Netlink treats the latter as a dump
 997                  * request.  We also accept NLM_F_EXCL in case that bug ever
 998                  * gets fixed.
 999                  */
1000                 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1001                                                          | NLM_F_EXCL))) {
1002                         error = -EEXIST;
1003                         goto err_unlock_ovs;
1004                 }
1005                 /* The flow identifier has to be the same for flow updates.
1006                  * Look for any overlapping flow.
1007                  */
1008                 if (unlikely(!ovs_flow_cmp(flow, &match))) {
1009                         if (ovs_identifier_is_key(&flow->id))
1010                                 flow = ovs_flow_tbl_lookup_exact(&dp->table,
1011                                                                  &match);
1012                         else /* UFID matches but key is different */
1013                                 flow = NULL;
1014                         if (!flow) {
1015                                 error = -ENOENT;
1016                                 goto err_unlock_ovs;
1017                         }
1018                 }
1019                 /* Update actions. */
1020                 old_acts = ovsl_dereference(flow->sf_acts);
1021                 rcu_assign_pointer(flow->sf_acts, acts);
1022
1023                 if (unlikely(reply)) {
1024                         error = ovs_flow_cmd_fill_info(flow,
1025                                                        ovs_header->dp_ifindex,
1026                                                        reply, info->snd_portid,
1027                                                        info->snd_seq, 0,
1028                                                        OVS_FLOW_CMD_NEW,
1029                                                        ufid_flags);
1030                         BUG_ON(error < 0);
1031                 }
1032                 ovs_unlock();
1033
1034                 ovs_nla_free_flow_actions_rcu(old_acts);
1035                 ovs_flow_free(new_flow, false);
1036         }
1037
1038         if (reply)
1039                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1040         return 0;
1041
1042 err_unlock_ovs:
1043         ovs_unlock();
1044         kfree_skb(reply);
1045 err_kfree_acts:
1046         ovs_nla_free_flow_actions(acts);
1047 err_kfree_flow:
1048         ovs_flow_free(new_flow, false);
1049 error:
1050         return error;
1051 }
1052
1053 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1054 static struct sw_flow_actions *get_flow_actions(const struct nlattr *a,
1055                                                 const struct sw_flow_key *key,
1056                                                 const struct sw_flow_mask *mask,
1057                                                 bool log)
1058 {
1059         struct sw_flow_actions *acts;
1060         struct sw_flow_key masked_key;
1061         int error;
1062
1063         ovs_flow_mask_key(&masked_key, key, true, mask);
1064         error = ovs_nla_copy_actions(a, &masked_key, &acts, log);
1065         if (error) {
1066                 OVS_NLERR(log,
1067                           "Actions may not be safe on all matching packets");
1068                 return ERR_PTR(error);
1069         }
1070
1071         return acts;
1072 }
1073
1074 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1075 {
1076         struct nlattr **a = info->attrs;
1077         struct ovs_header *ovs_header = info->userhdr;
1078         struct sw_flow_key key;
1079         struct sw_flow *flow;
1080         struct sw_flow_mask mask;
1081         struct sk_buff *reply = NULL;
1082         struct datapath *dp;
1083         struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1084         struct sw_flow_match match;
1085         struct sw_flow_id sfid;
1086         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1087         int error;
1088         bool log = !a[OVS_FLOW_ATTR_PROBE];
1089         bool ufid_present;
1090
1091         /* Extract key. */
1092         error = -EINVAL;
1093         if (!a[OVS_FLOW_ATTR_KEY]) {
1094                 OVS_NLERR(log, "Flow key attribute not present in set flow.");
1095                 goto error;
1096         }
1097
1098         ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1099         ovs_match_init(&match, &key, &mask);
1100         error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
1101                                   a[OVS_FLOW_ATTR_MASK], log);
1102         if (error)
1103                 goto error;
1104
1105         /* Validate actions. */
1106         if (a[OVS_FLOW_ATTR_ACTIONS]) {
1107                 acts = get_flow_actions(a[OVS_FLOW_ATTR_ACTIONS], &key, &mask,
1108                                         log);
1109                 if (IS_ERR(acts)) {
1110                         error = PTR_ERR(acts);
1111                         goto error;
1112                 }
1113
1114                 /* Can allocate before locking if have acts. */
1115                 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1116                                                 ufid_flags);
1117                 if (IS_ERR(reply)) {
1118                         error = PTR_ERR(reply);
1119                         goto err_kfree_acts;
1120                 }
1121         }
1122
1123         ovs_lock();
1124         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1125         if (unlikely(!dp)) {
1126                 error = -ENODEV;
1127                 goto err_unlock_ovs;
1128         }
1129         /* Check that the flow exists. */
1130         if (ufid_present)
1131                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1132         else
1133                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1134         if (unlikely(!flow)) {
1135                 error = -ENOENT;
1136                 goto err_unlock_ovs;
1137         }
1138
1139         /* Update actions, if present. */
1140         if (likely(acts)) {
1141                 old_acts = ovsl_dereference(flow->sf_acts);
1142                 rcu_assign_pointer(flow->sf_acts, acts);
1143
1144                 if (unlikely(reply)) {
1145                         error = ovs_flow_cmd_fill_info(flow,
1146                                                        ovs_header->dp_ifindex,
1147                                                        reply, info->snd_portid,
1148                                                        info->snd_seq, 0,
1149                                                        OVS_FLOW_CMD_NEW,
1150                                                        ufid_flags);
1151                         BUG_ON(error < 0);
1152                 }
1153         } else {
1154                 /* Could not alloc without acts before locking. */
1155                 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1156                                                 info, OVS_FLOW_CMD_NEW, false,
1157                                                 ufid_flags);
1158
1159                 if (unlikely(IS_ERR(reply))) {
1160                         error = PTR_ERR(reply);
1161                         goto err_unlock_ovs;
1162                 }
1163         }
1164
1165         /* Clear stats. */
1166         if (a[OVS_FLOW_ATTR_CLEAR])
1167                 ovs_flow_stats_clear(flow);
1168         ovs_unlock();
1169
1170         if (reply)
1171                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1172         if (old_acts)
1173                 ovs_nla_free_flow_actions_rcu(old_acts);
1174
1175         return 0;
1176
1177 err_unlock_ovs:
1178         ovs_unlock();
1179         kfree_skb(reply);
1180 err_kfree_acts:
1181         ovs_nla_free_flow_actions(acts);
1182 error:
1183         return error;
1184 }
1185
1186 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1187 {
1188         struct nlattr **a = info->attrs;
1189         struct ovs_header *ovs_header = info->userhdr;
1190         struct sw_flow_key key;
1191         struct sk_buff *reply;
1192         struct sw_flow *flow;
1193         struct datapath *dp;
1194         struct sw_flow_match match;
1195         struct sw_flow_id ufid;
1196         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1197         int err = 0;
1198         bool log = !a[OVS_FLOW_ATTR_PROBE];
1199         bool ufid_present;
1200
1201         ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1202         if (a[OVS_FLOW_ATTR_KEY]) {
1203                 ovs_match_init(&match, &key, NULL);
1204                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
1205                                         log);
1206         } else if (!ufid_present) {
1207                 OVS_NLERR(log,
1208                           "Flow get message rejected, Key attribute missing.");
1209                 err = -EINVAL;
1210         }
1211         if (err)
1212                 return err;
1213
1214         ovs_lock();
1215         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1216         if (!dp) {
1217                 err = -ENODEV;
1218                 goto unlock;
1219         }
1220
1221         if (ufid_present)
1222                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1223         else
1224                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1225         if (!flow) {
1226                 err = -ENOENT;
1227                 goto unlock;
1228         }
1229
1230         reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1231                                         OVS_FLOW_CMD_NEW, true, ufid_flags);
1232         if (IS_ERR(reply)) {
1233                 err = PTR_ERR(reply);
1234                 goto unlock;
1235         }
1236
1237         ovs_unlock();
1238         return genlmsg_reply(reply, info);
1239 unlock:
1240         ovs_unlock();
1241         return err;
1242 }
1243
1244 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1245 {
1246         struct nlattr **a = info->attrs;
1247         struct ovs_header *ovs_header = info->userhdr;
1248         struct sw_flow_key key;
1249         struct sk_buff *reply;
1250         struct sw_flow *flow = NULL;
1251         struct datapath *dp;
1252         struct sw_flow_match match;
1253         struct sw_flow_id ufid;
1254         u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1255         int err;
1256         bool log = !a[OVS_FLOW_ATTR_PROBE];
1257         bool ufid_present;
1258
1259         ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1260         if (a[OVS_FLOW_ATTR_KEY]) {
1261                 ovs_match_init(&match, &key, NULL);
1262                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
1263                                         log);
1264                 if (unlikely(err))
1265                         return err;
1266         }
1267
1268         ovs_lock();
1269         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1270         if (unlikely(!dp)) {
1271                 err = -ENODEV;
1272                 goto unlock;
1273         }
1274
1275         if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1276                 err = ovs_flow_tbl_flush(&dp->table);
1277                 goto unlock;
1278         }
1279
1280         if (ufid_present)
1281                 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1282         else
1283                 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1284         if (unlikely(!flow)) {
1285                 err = -ENOENT;
1286                 goto unlock;
1287         }
1288
1289         ovs_flow_tbl_remove(&dp->table, flow);
1290         ovs_unlock();
1291
1292         reply = ovs_flow_cmd_alloc_info(rcu_dereference_raw(flow->sf_acts),
1293                                         &flow->id, info, false, ufid_flags);
1294
1295         if (likely(reply)) {
1296                 if (likely(!IS_ERR(reply))) {
1297                         rcu_read_lock();        /*To keep RCU checker happy. */
1298                         err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1299                                                      reply, info->snd_portid,
1300                                                      info->snd_seq, 0,
1301                                                      OVS_FLOW_CMD_DEL,
1302                                                      ufid_flags);
1303                         rcu_read_unlock();
1304                         BUG_ON(err < 0);
1305                         ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1306                 } else {
1307                         genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1308                                      GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1309
1310                 }
1311         }
1312
1313         ovs_flow_free(flow, true);
1314         return 0;
1315 unlock:
1316         ovs_unlock();
1317         return err;
1318 }
1319
1320 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1321 {
1322         struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1323         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1324         struct table_instance *ti;
1325         struct datapath *dp;
1326         u32 ufid_flags;
1327         int err;
1328
1329         err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
1330                             OVS_FLOW_ATTR_MAX, flow_policy);
1331         if (err)
1332                 return err;
1333         ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1334
1335         rcu_read_lock();
1336         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1337         if (!dp) {
1338                 rcu_read_unlock();
1339                 return -ENODEV;
1340         }
1341
1342         ti = rcu_dereference(dp->table.ti);
1343         for (;;) {
1344                 struct sw_flow *flow;
1345                 u32 bucket, obj;
1346
1347                 bucket = cb->args[0];
1348                 obj = cb->args[1];
1349                 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1350                 if (!flow)
1351                         break;
1352
1353                 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1354                                            NETLINK_CB(cb->skb).portid,
1355                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1356                                            OVS_FLOW_CMD_NEW, ufid_flags) < 0)
1357                         break;
1358
1359                 cb->args[0] = bucket;
1360                 cb->args[1] = obj;
1361         }
1362         rcu_read_unlock();
1363         return skb->len;
1364 }
1365
1366 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1367         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1368         [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1369         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1370         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1371         [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1372         [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1373         [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1374 };
1375
1376 static struct genl_ops dp_flow_genl_ops[] = {
1377         { .cmd = OVS_FLOW_CMD_NEW,
1378           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1379           .policy = flow_policy,
1380           .doit = ovs_flow_cmd_new
1381         },
1382         { .cmd = OVS_FLOW_CMD_DEL,
1383           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1384           .policy = flow_policy,
1385           .doit = ovs_flow_cmd_del
1386         },
1387         { .cmd = OVS_FLOW_CMD_GET,
1388           .flags = 0,               /* OK for unprivileged users. */
1389           .policy = flow_policy,
1390           .doit = ovs_flow_cmd_get,
1391           .dumpit = ovs_flow_cmd_dump
1392         },
1393         { .cmd = OVS_FLOW_CMD_SET,
1394           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1395           .policy = flow_policy,
1396           .doit = ovs_flow_cmd_set,
1397         },
1398 };
1399
1400 static struct genl_family dp_flow_genl_family = {
1401         .id = GENL_ID_GENERATE,
1402         .hdrsize = sizeof(struct ovs_header),
1403         .name = OVS_FLOW_FAMILY,
1404         .version = OVS_FLOW_VERSION,
1405         .maxattr = OVS_FLOW_ATTR_MAX,
1406         .netnsok = true,
1407         .parallel_ops = true,
1408         .ops = dp_flow_genl_ops,
1409         .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1410         .mcgrps = &ovs_dp_flow_multicast_group,
1411         .n_mcgrps = 1,
1412 };
1413
1414 static size_t ovs_dp_cmd_msg_size(void)
1415 {
1416         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1417
1418         msgsize += nla_total_size(IFNAMSIZ);
1419         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1420         msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1421         msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1422
1423         return msgsize;
1424 }
1425
1426 /* Called with ovs_mutex. */
1427 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1428                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1429 {
1430         struct ovs_header *ovs_header;
1431         struct ovs_dp_stats dp_stats;
1432         struct ovs_dp_megaflow_stats dp_megaflow_stats;
1433         int err;
1434
1435         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1436                                    flags, cmd);
1437         if (!ovs_header)
1438                 goto error;
1439
1440         ovs_header->dp_ifindex = get_dpifindex(dp);
1441
1442         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1443         if (err)
1444                 goto nla_put_failure;
1445
1446         get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1447         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1448                         &dp_stats))
1449                 goto nla_put_failure;
1450
1451         if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1452                         sizeof(struct ovs_dp_megaflow_stats),
1453                         &dp_megaflow_stats))
1454                 goto nla_put_failure;
1455
1456         if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1457                 goto nla_put_failure;
1458
1459         genlmsg_end(skb, ovs_header);
1460         return 0;
1461
1462 nla_put_failure:
1463         genlmsg_cancel(skb, ovs_header);
1464 error:
1465         return -EMSGSIZE;
1466 }
1467
1468 static struct sk_buff *ovs_dp_cmd_alloc_info(struct genl_info *info)
1469 {
1470         return genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1471 }
1472
1473 /* Called with rcu_read_lock or ovs_mutex. */
1474 static struct datapath *lookup_datapath(struct net *net,
1475                                         const struct ovs_header *ovs_header,
1476                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1477 {
1478         struct datapath *dp;
1479
1480         if (!a[OVS_DP_ATTR_NAME])
1481                 dp = get_dp(net, ovs_header->dp_ifindex);
1482         else {
1483                 struct vport *vport;
1484
1485                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1486                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1487         }
1488         return dp ? dp : ERR_PTR(-ENODEV);
1489 }
1490
1491 static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1492 {
1493         struct datapath *dp;
1494
1495         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1496         if (IS_ERR(dp))
1497                 return;
1498
1499         WARN(dp->user_features, "Dropping previously announced user features\n");
1500         dp->user_features = 0;
1501 }
1502
1503 static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1504 {
1505         if (a[OVS_DP_ATTR_USER_FEATURES])
1506                 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1507 }
1508
1509 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1510 {
1511         struct nlattr **a = info->attrs;
1512         struct vport_parms parms;
1513         struct sk_buff *reply;
1514         struct datapath *dp;
1515         struct vport *vport;
1516         struct ovs_net *ovs_net;
1517         int err, i;
1518
1519         err = -EINVAL;
1520         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1521                 goto err;
1522
1523         reply = ovs_dp_cmd_alloc_info(info);
1524         if (!reply)
1525                 return -ENOMEM;
1526
1527         err = -ENOMEM;
1528         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1529         if (dp == NULL)
1530                 goto err_free_reply;
1531
1532         ovs_dp_set_net(dp, sock_net(skb->sk));
1533
1534         /* Allocate table. */
1535         err = ovs_flow_tbl_init(&dp->table);
1536         if (err)
1537                 goto err_free_dp;
1538
1539         dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1540         if (!dp->stats_percpu) {
1541                 err = -ENOMEM;
1542                 goto err_destroy_table;
1543         }
1544
1545         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1546                             GFP_KERNEL);
1547         if (!dp->ports) {
1548                 err = -ENOMEM;
1549                 goto err_destroy_percpu;
1550         }
1551
1552         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1553                 INIT_HLIST_HEAD(&dp->ports[i]);
1554
1555         /* Set up our datapath device. */
1556         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1557         parms.type = OVS_VPORT_TYPE_INTERNAL;
1558         parms.options = NULL;
1559         parms.dp = dp;
1560         parms.port_no = OVSP_LOCAL;
1561         parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1562
1563         ovs_dp_change(dp, a);
1564
1565         /* So far only local changes have been made, now need the lock. */
1566         ovs_lock();
1567
1568         vport = new_vport(&parms);
1569         if (IS_ERR(vport)) {
1570                 err = PTR_ERR(vport);
1571                 if (err == -EBUSY)
1572                         err = -EEXIST;
1573
1574                 if (err == -EEXIST) {
1575                         /* An outdated user space instance that does not understand
1576                          * the concept of user_features has attempted to create a new
1577                          * datapath and is likely to reuse it. Drop all user features.
1578                          */
1579                         if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1580                                 ovs_dp_reset_user_features(skb, info);
1581                 }
1582
1583                 goto err_destroy_ports_array;
1584         }
1585
1586         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1587                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1588         BUG_ON(err < 0);
1589
1590         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1591         list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1592
1593         ovs_unlock();
1594
1595         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1596         return 0;
1597
1598 err_destroy_ports_array:
1599         ovs_unlock();
1600         kfree(dp->ports);
1601 err_destroy_percpu:
1602         free_percpu(dp->stats_percpu);
1603 err_destroy_table:
1604         ovs_flow_tbl_destroy(&dp->table);
1605 err_free_dp:
1606         kfree(dp);
1607 err_free_reply:
1608         kfree_skb(reply);
1609 err:
1610         return err;
1611 }
1612
1613 /* Called with ovs_mutex. */
1614 static void __dp_destroy(struct datapath *dp)
1615 {
1616         int i;
1617
1618         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1619                 struct vport *vport;
1620                 struct hlist_node *n;
1621
1622                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1623                         if (vport->port_no != OVSP_LOCAL)
1624                                 ovs_dp_detach_port(vport);
1625         }
1626
1627         list_del_rcu(&dp->list_node);
1628
1629         /* OVSP_LOCAL is datapath internal port. We need to make sure that
1630          * all ports in datapath are destroyed first before freeing datapath.
1631          */
1632         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1633
1634         /* RCU destroy the flow table */
1635         call_rcu(&dp->rcu, destroy_dp_rcu);
1636 }
1637
1638 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1639 {
1640         struct sk_buff *reply;
1641         struct datapath *dp;
1642         int err;
1643
1644         reply = ovs_dp_cmd_alloc_info(info);
1645         if (!reply)
1646                 return -ENOMEM;
1647
1648         ovs_lock();
1649         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1650         err = PTR_ERR(dp);
1651         if (IS_ERR(dp))
1652                 goto err_unlock_free;
1653
1654         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1655                                    info->snd_seq, 0, OVS_DP_CMD_DEL);
1656         BUG_ON(err < 0);
1657
1658         __dp_destroy(dp);
1659         ovs_unlock();
1660
1661         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1662         return 0;
1663
1664 err_unlock_free:
1665         ovs_unlock();
1666         kfree_skb(reply);
1667         return err;
1668 }
1669
1670 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1671 {
1672         struct sk_buff *reply;
1673         struct datapath *dp;
1674         int err;
1675
1676         reply = ovs_dp_cmd_alloc_info(info);
1677         if (!reply)
1678                 return -ENOMEM;
1679
1680         ovs_lock();
1681         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1682         err = PTR_ERR(dp);
1683         if (IS_ERR(dp))
1684                 goto err_unlock_free;
1685
1686         ovs_dp_change(dp, info->attrs);
1687
1688         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1689                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1690         BUG_ON(err < 0);
1691
1692         ovs_unlock();
1693
1694         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1695         return 0;
1696
1697 err_unlock_free:
1698         ovs_unlock();
1699         kfree_skb(reply);
1700         return err;
1701 }
1702
1703 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1704 {
1705         struct sk_buff *reply;
1706         struct datapath *dp;
1707         int err;
1708
1709         reply = ovs_dp_cmd_alloc_info(info);
1710         if (!reply)
1711                 return -ENOMEM;
1712
1713         ovs_lock();
1714         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1715         if (IS_ERR(dp)) {
1716                 err = PTR_ERR(dp);
1717                 goto err_unlock_free;
1718         }
1719         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1720                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1721         BUG_ON(err < 0);
1722         ovs_unlock();
1723
1724         return genlmsg_reply(reply, info);
1725
1726 err_unlock_free:
1727         ovs_unlock();
1728         kfree_skb(reply);
1729         return err;
1730 }
1731
1732 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1733 {
1734         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1735         struct datapath *dp;
1736         int skip = cb->args[0];
1737         int i = 0;
1738
1739         ovs_lock();
1740         list_for_each_entry(dp, &ovs_net->dps, list_node) {
1741                 if (i >= skip &&
1742                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1743                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1744                                          OVS_DP_CMD_NEW) < 0)
1745                         break;
1746                 i++;
1747         }
1748         ovs_unlock();
1749
1750         cb->args[0] = i;
1751
1752         return skb->len;
1753 }
1754
1755 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1756         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1757         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1758         [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1759 };
1760
1761 static struct genl_ops dp_datapath_genl_ops[] = {
1762         { .cmd = OVS_DP_CMD_NEW,
1763           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1764           .policy = datapath_policy,
1765           .doit = ovs_dp_cmd_new
1766         },
1767         { .cmd = OVS_DP_CMD_DEL,
1768           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1769           .policy = datapath_policy,
1770           .doit = ovs_dp_cmd_del
1771         },
1772         { .cmd = OVS_DP_CMD_GET,
1773           .flags = 0,               /* OK for unprivileged users. */
1774           .policy = datapath_policy,
1775           .doit = ovs_dp_cmd_get,
1776           .dumpit = ovs_dp_cmd_dump
1777         },
1778         { .cmd = OVS_DP_CMD_SET,
1779           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1780           .policy = datapath_policy,
1781           .doit = ovs_dp_cmd_set,
1782         },
1783 };
1784
1785 static struct genl_family dp_datapath_genl_family = {
1786         .id = GENL_ID_GENERATE,
1787         .hdrsize = sizeof(struct ovs_header),
1788         .name = OVS_DATAPATH_FAMILY,
1789         .version = OVS_DATAPATH_VERSION,
1790         .maxattr = OVS_DP_ATTR_MAX,
1791         .netnsok = true,
1792         .parallel_ops = true,
1793         .ops = dp_datapath_genl_ops,
1794         .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1795         .mcgrps = &ovs_dp_datapath_multicast_group,
1796         .n_mcgrps = 1,
1797 };
1798
1799 /* Called with ovs_mutex or RCU read lock. */
1800 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1801                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1802 {
1803         struct ovs_header *ovs_header;
1804         struct ovs_vport_stats vport_stats;
1805         int err;
1806
1807         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1808                                  flags, cmd);
1809         if (!ovs_header)
1810                 return -EMSGSIZE;
1811
1812         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1813
1814         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1815             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1816             nla_put_string(skb, OVS_VPORT_ATTR_NAME,
1817                            ovs_vport_name(vport)))
1818                 goto nla_put_failure;
1819
1820         ovs_vport_get_stats(vport, &vport_stats);
1821         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1822                     &vport_stats))
1823                 goto nla_put_failure;
1824
1825         if (ovs_vport_get_upcall_portids(vport, skb))
1826                 goto nla_put_failure;
1827
1828         err = ovs_vport_get_options(vport, skb);
1829         if (err == -EMSGSIZE)
1830                 goto error;
1831
1832         genlmsg_end(skb, ovs_header);
1833         return 0;
1834
1835 nla_put_failure:
1836         err = -EMSGSIZE;
1837 error:
1838         genlmsg_cancel(skb, ovs_header);
1839         return err;
1840 }
1841
1842 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1843 {
1844         return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1845 }
1846
1847 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1848 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1849                                          u32 seq, u8 cmd)
1850 {
1851         struct sk_buff *skb;
1852         int retval;
1853
1854         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1855         if (!skb)
1856                 return ERR_PTR(-ENOMEM);
1857
1858         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1859         BUG_ON(retval < 0);
1860
1861         return skb;
1862 }
1863
1864 /* Called with ovs_mutex or RCU read lock. */
1865 static struct vport *lookup_vport(struct net *net,
1866                                   const struct ovs_header *ovs_header,
1867                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1868 {
1869         struct datapath *dp;
1870         struct vport *vport;
1871
1872         if (a[OVS_VPORT_ATTR_NAME]) {
1873                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1874                 if (!vport)
1875                         return ERR_PTR(-ENODEV);
1876                 if (ovs_header->dp_ifindex &&
1877                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1878                         return ERR_PTR(-ENODEV);
1879                 return vport;
1880         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1881                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1882
1883                 if (port_no >= DP_MAX_PORTS)
1884                         return ERR_PTR(-EFBIG);
1885
1886                 dp = get_dp(net, ovs_header->dp_ifindex);
1887                 if (!dp)
1888                         return ERR_PTR(-ENODEV);
1889
1890                 vport = ovs_vport_ovsl_rcu(dp, port_no);
1891                 if (!vport)
1892                         return ERR_PTR(-ENODEV);
1893                 return vport;
1894         } else
1895                 return ERR_PTR(-EINVAL);
1896 }
1897
1898 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1899 {
1900         struct nlattr **a = info->attrs;
1901         struct ovs_header *ovs_header = info->userhdr;
1902         struct vport_parms parms;
1903         struct sk_buff *reply;
1904         struct vport *vport;
1905         struct datapath *dp;
1906         u32 port_no;
1907         int err;
1908
1909         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1910             !a[OVS_VPORT_ATTR_UPCALL_PID])
1911                 return -EINVAL;
1912
1913         port_no = a[OVS_VPORT_ATTR_PORT_NO]
1914                 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
1915         if (port_no >= DP_MAX_PORTS)
1916                 return -EFBIG;
1917
1918         reply = ovs_vport_cmd_alloc_info();
1919         if (!reply)
1920                 return -ENOMEM;
1921
1922         ovs_lock();
1923 restart:
1924         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1925         err = -ENODEV;
1926         if (!dp)
1927                 goto exit_unlock_free;
1928
1929         if (port_no) {
1930                 vport = ovs_vport_ovsl(dp, port_no);
1931                 err = -EBUSY;
1932                 if (vport)
1933                         goto exit_unlock_free;
1934         } else {
1935                 for (port_no = 1; ; port_no++) {
1936                         if (port_no >= DP_MAX_PORTS) {
1937                                 err = -EFBIG;
1938                                 goto exit_unlock_free;
1939                         }
1940                         vport = ovs_vport_ovsl(dp, port_no);
1941                         if (!vport)
1942                                 break;
1943                 }
1944         }
1945
1946         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1947         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1948         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1949         parms.dp = dp;
1950         parms.port_no = port_no;
1951         parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
1952
1953         vport = new_vport(&parms);
1954         err = PTR_ERR(vport);
1955         if (IS_ERR(vport)) {
1956                 if (err == -EAGAIN)
1957                         goto restart;
1958                 goto exit_unlock_free;
1959         }
1960
1961         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1962                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1963         BUG_ON(err < 0);
1964         ovs_unlock();
1965
1966         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1967         return 0;
1968
1969 exit_unlock_free:
1970         ovs_unlock();
1971         kfree_skb(reply);
1972         return err;
1973 }
1974
1975 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
1976 {
1977         struct nlattr **a = info->attrs;
1978         struct sk_buff *reply;
1979         struct vport *vport;
1980         int err;
1981
1982         reply = ovs_vport_cmd_alloc_info();
1983         if (!reply)
1984                 return -ENOMEM;
1985
1986         ovs_lock();
1987         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1988         err = PTR_ERR(vport);
1989         if (IS_ERR(vport))
1990                 goto exit_unlock_free;
1991
1992         if (a[OVS_VPORT_ATTR_TYPE] &&
1993             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1994                 err = -EINVAL;
1995                 goto exit_unlock_free;
1996         }
1997
1998         if (a[OVS_VPORT_ATTR_OPTIONS]) {
1999                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2000                 if (err)
2001                         goto exit_unlock_free;
2002         }
2003
2004         if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2005                 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2006
2007                 err = ovs_vport_set_upcall_portids(vport, ids);
2008                 if (err)
2009                         goto exit_unlock_free;
2010         }
2011
2012         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2013                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2014         BUG_ON(err < 0);
2015         ovs_unlock();
2016
2017         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
2018         return 0;
2019
2020 exit_unlock_free:
2021         ovs_unlock();
2022         kfree_skb(reply);
2023         return err;
2024 }
2025
2026 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2027 {
2028         struct nlattr **a = info->attrs;
2029         struct sk_buff *reply;
2030         struct vport *vport;
2031         int err;
2032
2033         reply = ovs_vport_cmd_alloc_info();
2034         if (!reply)
2035                 return -ENOMEM;
2036
2037         ovs_lock();
2038         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2039         err = PTR_ERR(vport);
2040         if (IS_ERR(vport))
2041                 goto exit_unlock_free;
2042
2043         if (vport->port_no == OVSP_LOCAL) {
2044                 err = -EINVAL;
2045                 goto exit_unlock_free;
2046         }
2047
2048         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2049                                       info->snd_seq, 0, OVS_VPORT_CMD_DEL);
2050         BUG_ON(err < 0);
2051         ovs_dp_detach_port(vport);
2052         ovs_unlock();
2053
2054         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
2055         return 0;
2056
2057 exit_unlock_free:
2058         ovs_unlock();
2059         kfree_skb(reply);
2060         return err;
2061 }
2062
2063 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2064 {
2065         struct nlattr **a = info->attrs;
2066         struct ovs_header *ovs_header = info->userhdr;
2067         struct sk_buff *reply;
2068         struct vport *vport;
2069         int err;
2070
2071         reply = ovs_vport_cmd_alloc_info();
2072         if (!reply)
2073                 return -ENOMEM;
2074
2075         rcu_read_lock();
2076         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2077         err = PTR_ERR(vport);
2078         if (IS_ERR(vport))
2079                 goto exit_unlock_free;
2080         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2081                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2082         BUG_ON(err < 0);
2083         rcu_read_unlock();
2084
2085         return genlmsg_reply(reply, info);
2086
2087 exit_unlock_free:
2088         rcu_read_unlock();
2089         kfree_skb(reply);
2090         return err;
2091 }
2092
2093 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2094 {
2095         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2096         struct datapath *dp;
2097         int bucket = cb->args[0], skip = cb->args[1];
2098         int i, j = 0;
2099
2100         rcu_read_lock();
2101         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2102         if (!dp) {
2103                 rcu_read_unlock();
2104                 return -ENODEV;
2105         }
2106         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2107                 struct vport *vport;
2108
2109                 j = 0;
2110                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2111                         if (j >= skip &&
2112                             ovs_vport_cmd_fill_info(vport, skb,
2113                                                     NETLINK_CB(cb->skb).portid,
2114                                                     cb->nlh->nlmsg_seq,
2115                                                     NLM_F_MULTI,
2116                                                     OVS_VPORT_CMD_NEW) < 0)
2117                                 goto out;
2118
2119                         j++;
2120                 }
2121                 skip = 0;
2122         }
2123 out:
2124         rcu_read_unlock();
2125
2126         cb->args[0] = i;
2127         cb->args[1] = j;
2128
2129         return skb->len;
2130 }
2131
2132 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2133         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2134         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2135         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2136         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2137         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2138         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2139 };
2140
2141 static struct genl_ops dp_vport_genl_ops[] = {
2142         { .cmd = OVS_VPORT_CMD_NEW,
2143           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2144           .policy = vport_policy,
2145           .doit = ovs_vport_cmd_new
2146         },
2147         { .cmd = OVS_VPORT_CMD_DEL,
2148           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2149           .policy = vport_policy,
2150           .doit = ovs_vport_cmd_del
2151         },
2152         { .cmd = OVS_VPORT_CMD_GET,
2153           .flags = 0,               /* OK for unprivileged users. */
2154           .policy = vport_policy,
2155           .doit = ovs_vport_cmd_get,
2156           .dumpit = ovs_vport_cmd_dump
2157         },
2158         { .cmd = OVS_VPORT_CMD_SET,
2159           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2160           .policy = vport_policy,
2161           .doit = ovs_vport_cmd_set,
2162         },
2163 };
2164
2165 struct genl_family dp_vport_genl_family = {
2166         .id = GENL_ID_GENERATE,
2167         .hdrsize = sizeof(struct ovs_header),
2168         .name = OVS_VPORT_FAMILY,
2169         .version = OVS_VPORT_VERSION,
2170         .maxattr = OVS_VPORT_ATTR_MAX,
2171         .netnsok = true,
2172         .parallel_ops = true,
2173         .ops = dp_vport_genl_ops,
2174         .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2175         .mcgrps = &ovs_dp_vport_multicast_group,
2176         .n_mcgrps = 1,
2177 };
2178
2179 static struct genl_family *dp_genl_families[] = {
2180         &dp_datapath_genl_family,
2181         &dp_vport_genl_family,
2182         &dp_flow_genl_family,
2183         &dp_packet_genl_family,
2184 };
2185
2186 static void dp_unregister_genl(int n_families)
2187 {
2188         int i;
2189
2190         for (i = 0; i < n_families; i++)
2191                 genl_unregister_family(dp_genl_families[i]);
2192 }
2193
2194 static int dp_register_genl(void)
2195 {
2196         int err;
2197         int i;
2198
2199         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2200
2201                 err = genl_register_family(dp_genl_families[i]);
2202                 if (err)
2203                         goto error;
2204         }
2205
2206         return 0;
2207
2208 error:
2209         dp_unregister_genl(i);
2210         return err;
2211 }
2212
2213 static int __net_init ovs_init_net(struct net *net)
2214 {
2215         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2216
2217         INIT_LIST_HEAD(&ovs_net->dps);
2218         INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2219         return 0;
2220 }
2221
2222 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2223                                             struct list_head *head)
2224 {
2225         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2226         struct datapath *dp;
2227
2228         list_for_each_entry(dp, &ovs_net->dps, list_node) {
2229                 int i;
2230
2231                 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2232                         struct vport *vport;
2233
2234                         hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2235
2236                                 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2237                                         continue;
2238
2239                                 if (dev_net(vport->dev) == dnet)
2240                                         list_add(&vport->detach_list, head);
2241                         }
2242                 }
2243         }
2244 }
2245
2246 static void __net_exit ovs_exit_net(struct net *dnet)
2247 {
2248         struct datapath *dp, *dp_next;
2249         struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2250         struct vport *vport, *vport_next;
2251         struct net *net;
2252         LIST_HEAD(head);
2253
2254         ovs_lock();
2255         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2256                 __dp_destroy(dp);
2257
2258         rtnl_lock();
2259         for_each_net(net)
2260                 list_vports_from_net(net, dnet, &head);
2261         rtnl_unlock();
2262
2263         /* Detach all vports from given namespace. */
2264         list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2265                 list_del(&vport->detach_list);
2266                 ovs_dp_detach_port(vport);
2267         }
2268
2269         ovs_unlock();
2270
2271         cancel_work_sync(&ovs_net->dp_notify_work);
2272 }
2273
2274 static struct pernet_operations ovs_net_ops = {
2275         .init = ovs_init_net,
2276         .exit = ovs_exit_net,
2277         .id   = &ovs_net_id,
2278         .size = sizeof(struct ovs_net),
2279 };
2280
2281 DEFINE_COMPAT_PNET_REG_FUNC(device);
2282
2283 static int __init dp_init(void)
2284 {
2285         int err;
2286
2287         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2288
2289         pr_info("Open vSwitch switching datapath %s\n", VERSION);
2290
2291         err = action_fifos_init();
2292         if (err)
2293                 goto error;
2294
2295         err = ovs_internal_dev_rtnl_link_register();
2296         if (err)
2297                 goto error_action_fifos_exit;
2298
2299         err = ovs_flow_init();
2300         if (err)
2301                 goto error_unreg_rtnl_link;
2302
2303         err = ovs_vport_init();
2304         if (err)
2305                 goto error_flow_exit;
2306
2307         err = register_pernet_device(&ovs_net_ops);
2308         if (err)
2309                 goto error_vport_exit;
2310
2311         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2312         if (err)
2313                 goto error_netns_exit;
2314
2315         err = ovs_netdev_init();
2316         if (err)
2317                 goto error_unreg_notifier;
2318
2319         err = dp_register_genl();
2320         if (err < 0)
2321                 goto error_unreg_netdev;
2322
2323         return 0;
2324
2325 error_unreg_netdev:
2326         ovs_netdev_exit();
2327 error_unreg_notifier:
2328         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2329 error_netns_exit:
2330         unregister_pernet_device(&ovs_net_ops);
2331 error_vport_exit:
2332         ovs_vport_exit();
2333 error_flow_exit:
2334         ovs_flow_exit();
2335 error_unreg_rtnl_link:
2336         ovs_internal_dev_rtnl_link_unregister();
2337 error_action_fifos_exit:
2338         action_fifos_exit();
2339 error:
2340         return err;
2341 }
2342
2343 static void dp_cleanup(void)
2344 {
2345         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2346         ovs_netdev_exit();
2347         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2348         unregister_pernet_device(&ovs_net_ops);
2349         rcu_barrier();
2350         ovs_vport_exit();
2351         ovs_flow_exit();
2352         ovs_internal_dev_rtnl_link_unregister();
2353         action_fifos_exit();
2354 }
2355
2356 module_init(dp_init);
2357 module_exit(dp_cleanup);
2358
2359 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2360 MODULE_LICENSE("GPL");
2361 MODULE_VERSION(VERSION);