datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2014 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <linux/genetlink.h>
  52 #include <net/genetlink.h>
  53 #include <net/genetlink.h>
  54 #include <net/net_namespace.h>
  55 #include <net/netns/generic.h>
  56
  57 #include "datapath.h"
  58 #include "flow.h"
  59 #include "flow_table.h"
  60 #include "flow_netlink.h"
  61 #include "vlan.h"
  62 #include "vport-internal_dev.h"
  63 #include "vport-netdev.h"
  64
  65 int ovs_net_id __read_mostly;
  66
  67 static struct genl_family dp_packet_genl_family;
  68 static struct genl_family dp_flow_genl_family;
  69 static struct genl_family dp_datapath_genl_family;
  70
  71 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
  72         .name = OVS_FLOW_MCGROUP
  73 };
  74
  75 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  76         .name = OVS_DATAPATH_MCGROUP
  77 };
  78
  79 struct genl_multicast_group ovs_dp_vport_multicast_group = {
  80         .name = OVS_VPORT_MCGROUP
  81 };
  82
  83 /* Check if need to build a reply message.
  84  * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
  85 static bool ovs_must_notify(struct genl_info *info,
  86                             const struct genl_multicast_group *grp)
  87 {
  88         return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  89                 netlink_has_listeners(genl_info_net(info)->genl_sock, GROUP_ID(grp));
  90 }
  91
  92 static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
  93                        struct sk_buff *skb, struct genl_info *info)
  94 {
  95         genl_notify(family, skb, genl_info_net(info),
  96                     info->snd_portid, GROUP_ID(grp), info->nlhdr, GFP_KERNEL);
  97 }
  98
  99 /**
 100  * DOC: Locking:
 101  *
 102  * All writes e.g. Writes to device state (add/remove datapath, port, set
 103  * operations on vports, etc.), Writes to other state (flow table
 104  * modifications, set miscellaneous datapath parameters, etc.) are protected
 105  * by ovs_lock.
 106  *
 107  * Reads are protected by RCU.
 108  *
 109  * There are a few special cases (mostly stats) that have their own
 110  * synchronization but they nest under all of above and don't interact with
 111  * each other.
 112  *
 113  * The RTNL lock nests inside ovs_mutex.
 114  */
 115
 116 static DEFINE_MUTEX(ovs_mutex);
 117
 118 void ovs_lock(void)
 119 {
 120         mutex_lock(&ovs_mutex);
 121 }
 122
 123 void ovs_unlock(void)
 124 {
 125         mutex_unlock(&ovs_mutex);
 126 }
 127
 128 #ifdef CONFIG_LOCKDEP
 129 int lockdep_ovsl_is_held(void)
 130 {
 131         if (debug_locks)
 132                 return lockdep_is_held(&ovs_mutex);
 133         else
 134                 return 1;
 135 }
 136 #endif
 137
 138 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 139                              const struct dp_upcall_info *);
 140 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 141                                   const struct dp_upcall_info *);
 142
 143 /* Must be called with rcu_read_lock or ovs_mutex. */
 144 static struct datapath *get_dp(struct net *net, int dp_ifindex)
 145 {
 146         struct datapath *dp = NULL;
 147         struct net_device *dev;
 148
 149         rcu_read_lock();
 150         dev = dev_get_by_index_rcu(net, dp_ifindex);
 151         if (dev) {
 152                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 153                 if (vport)
 154                         dp = vport->dp;
 155         }
 156         rcu_read_unlock();
 157
 158         return dp;
 159 }
 160
 161 /* Must be called with rcu_read_lock or ovs_mutex. */
 162 const char *ovs_dp_name(const struct datapath *dp)
 163 {
 164         struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 165         return vport->ops->get_name(vport);
 166 }
 167
 168 static int get_dpifindex(struct datapath *dp)
 169 {
 170         struct vport *local;
 171         int ifindex;
 172
 173         rcu_read_lock();
 174
 175         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 176         if (local)
 177                 ifindex = netdev_vport_priv(local)->dev->ifindex;
 178         else
 179                 ifindex = 0;
 180
 181         rcu_read_unlock();
 182
 183         return ifindex;
 184 }
 185
 186 static void destroy_dp_rcu(struct rcu_head *rcu)
 187 {
 188         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 189
 190         ovs_flow_tbl_destroy(&dp->table);
 191         free_percpu(dp->stats_percpu);
 192         release_net(ovs_dp_get_net(dp));
 193         kfree(dp->ports);
 194         kfree(dp);
 195 }
 196
 197 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 198                                             u16 port_no)
 199 {
 200         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 201 }
 202
 203 /* Called with ovs_mutex or RCU read lock. */
 204 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 205 {
 206         struct vport *vport;
 207         struct hlist_head *head;
 208
 209         head = vport_hash_bucket(dp, port_no);
 210         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 211                 if (vport->port_no == port_no)
 212                         return vport;
 213         }
 214         return NULL;
 215 }
 216
 217 /* Called with ovs_mutex. */
 218 static struct vport *new_vport(const struct vport_parms *parms)
 219 {
 220         struct vport *vport;
 221
 222         vport = ovs_vport_add(parms);
 223         if (!IS_ERR(vport)) {
 224                 struct datapath *dp = parms->dp;
 225                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 226
 227                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 228         }
 229         return vport;
 230 }
 231
 232 void ovs_dp_detach_port(struct vport *p)
 233 {
 234         ASSERT_OVSL();
 235
 236         /* First drop references to device. */
 237         hlist_del_rcu(&p->dp_hash_node);
 238
 239         /* Then destroy it. */
 240         ovs_vport_del(p);
 241 }
 242
 243 void ovs_dp_process_packet_with_key(struct sk_buff *skb,
 244                                     struct sw_flow_key *pkt_key,
 245                                     bool recirc)
 246 {
 247         const struct vport *p = OVS_CB(skb)->input_vport;
 248         struct datapath *dp = p->dp;
 249         struct sw_flow *flow;
 250         struct dp_stats_percpu *stats;
 251         u64 *stats_counter;
 252         u32 n_mask_hit;
 253
 254         stats = this_cpu_ptr(dp->stats_percpu);
 255
 256         /* Look up flow. */
 257         flow = ovs_flow_tbl_lookup_stats(&dp->table, pkt_key, skb_get_hash(skb),
 258                                          &n_mask_hit);
 259         if (unlikely(!flow)) {
 260                 struct dp_upcall_info upcall;
 261
 262                 upcall.cmd = OVS_PACKET_CMD_MISS;
 263                 upcall.key = pkt_key;
 264                 upcall.userdata = NULL;
 265                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 266                 ovs_dp_upcall(dp, skb, &upcall);
 267                 consume_skb(skb);
 268                 stats_counter = &stats->n_missed;
 269                 goto out;
 270         }
 271
 272         OVS_CB(skb)->pkt_key = pkt_key;
 273         OVS_CB(skb)->flow = flow;
 274
 275         ovs_flow_stats_update(OVS_CB(skb)->flow, pkt_key->tp.flags, skb);
 276         ovs_execute_actions(dp, skb, recirc);
 277         stats_counter = &stats->n_hit;
 278
 279 out:
 280         /* Update datapath statistics. */
 281         u64_stats_update_begin(&stats->sync);
 282         (*stats_counter)++;
 283         stats->n_mask_hit += n_mask_hit;
 284         u64_stats_update_end(&stats->sync);
 285 }
 286
 287 /* Must be called with rcu_read_lock. */
 288 void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
 289 {
 290         int error;
 291         struct sw_flow_key key;
 292
 293         OVS_CB(skb)->input_vport = p;
 294
 295         /* Extract flow from 'skb' into 'key'. */
 296         error = ovs_flow_key_extract(skb, &key);
 297         if (unlikely(error)) {
 298                 kfree_skb(skb);
 299                 return;
 300         }
 301
 302         ovs_dp_process_packet_with_key(skb, &key, false);
 303 }
 304
 305 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 306                   const struct dp_upcall_info *upcall_info)
 307 {
 308         struct dp_stats_percpu *stats;
 309         int err;
 310
 311         if (upcall_info->portid == 0) {
 312                 err = -ENOTCONN;
 313                 goto err;
 314         }
 315
 316         if (!skb_is_gso(skb))
 317                 err = queue_userspace_packet(dp, skb, upcall_info);
 318         else
 319                 err = queue_gso_packets(dp, skb, upcall_info);
 320         if (err)
 321                 goto err;
 322
 323         return 0;
 324
 325 err:
 326         stats = this_cpu_ptr(dp->stats_percpu);
 327
 328         u64_stats_update_begin(&stats->sync);
 329         stats->n_lost++;
 330         u64_stats_update_end(&stats->sync);
 331
 332         return err;
 333 }
 334
 335 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 336                              const struct dp_upcall_info *upcall_info)
 337 {
 338         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 339         struct dp_upcall_info later_info;
 340         struct sw_flow_key later_key;
 341         struct sk_buff *segs, *nskb;
 342         struct ovs_skb_cb ovs_cb;
 343         int err;
 344
 345         ovs_cb = *OVS_CB(skb);
 346         segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 347         *OVS_CB(skb) = ovs_cb;
 348         if (IS_ERR(segs))
 349                 return PTR_ERR(segs);
 350
 351         /* Queue all of the segments. */
 352         skb = segs;
 353         do {
 354                 *OVS_CB(skb) = ovs_cb;
 355                 err = queue_userspace_packet(dp, skb, upcall_info);
 356                 if (err)
 357                         break;
 358
 359                 if (skb == segs && gso_type & SKB_GSO_UDP) {
 360                         /* The initial flow key extracted by ovs_flow_extract()
 361                          * in this case is for a first fragment, so we need to
 362                          * properly mark later fragments.
 363                          */
 364                         later_key = *upcall_info->key;
 365                         later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 366
 367                         later_info = *upcall_info;
 368                         later_info.key = &later_key;
 369                         upcall_info = &later_info;
 370                 }
 371         } while ((skb = skb->next));
 372
 373         /* Free all of the segments. */
 374         skb = segs;
 375         do {
 376                 nskb = skb->next;
 377                 if (err)
 378                         kfree_skb(skb);
 379                 else
 380                         consume_skb(skb);
 381         } while ((skb = nskb));
 382         return err;
 383 }
 384
 385 static size_t key_attr_size(void)
 386 {
 387         /* Whenever adding new OVS_KEY_ FIELDS, we should consider
 388          * updating this function.  */
 389         BUILD_BUG_ON(OVS_KEY_ATTR_IPV4_TUNNEL != 21);
 390
 391         return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 392                 + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
 393                   + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
 394                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
 395                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
 396                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
 397                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
 398                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
 399                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
 400                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 401                 + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 402                 + nla_total_size(4)   /* OVS_KEY_ATTR_DP_HASH */
 403                 + nla_total_size(4)   /* OVS_KEY_ATTR_RECIRC_ID */
 404                 + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
 405                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 406                 + nla_total_size(4)   /* OVS_KEY_ATTR_8021Q */
 407                 + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
 408                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 409                 + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
 410                 + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
 411                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 412 }
 413
 414 static size_t upcall_msg_size(const struct nlattr *userdata,
 415                               unsigned int hdrlen)
 416 {
 417         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 418                 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 419                 + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 420
 421         /* OVS_PACKET_ATTR_USERDATA */
 422         if (userdata)
 423                 size += NLA_ALIGN(userdata->nla_len);
 424
 425         return size;
 426 }
 427
 428 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 429                                   const struct dp_upcall_info *upcall_info)
 430 {
 431         struct ovs_header *upcall;
 432         struct sk_buff *nskb = NULL;
 433         struct sk_buff *user_skb; /* to be queued to userspace */
 434         struct nlattr *nla;
 435         struct genl_info info = {
 436 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
 437                 .dst_sk = ovs_dp_get_net(dp)->genl_sock,
 438 #endif
 439                 .snd_portid = upcall_info->portid,
 440         };
 441         size_t len;
 442         unsigned int hlen;
 443         int err, dp_ifindex;
 444
 445         dp_ifindex = get_dpifindex(dp);
 446         if (!dp_ifindex)
 447                 return -ENODEV;
 448
 449         if (vlan_tx_tag_present(skb)) {
 450                 nskb = skb_clone(skb, GFP_ATOMIC);
 451                 if (!nskb)
 452                         return -ENOMEM;
 453
 454                 nskb = __vlan_put_tag(nskb, nskb->vlan_proto, vlan_tx_tag_get(nskb));
 455                 if (!nskb)
 456                         return -ENOMEM;
 457
 458                 vlan_set_tci(nskb, 0);
 459
 460                 skb = nskb;
 461         }
 462
 463         if (nla_attr_size(skb->len) > USHRT_MAX) {
 464                 err = -EFBIG;
 465                 goto out;
 466         }
 467
 468         /* Complete checksum if needed */
 469         if (skb->ip_summed == CHECKSUM_PARTIAL &&
 470             (err = skb_checksum_help(skb)))
 471                 goto out;
 472
 473         /* Older versions of OVS user space enforce alignment of the last
 474          * Netlink attribute to NLA_ALIGNTO which would require extensive
 475          * padding logic. Only perform zerocopy if padding is not required.
 476          */
 477         if (dp->user_features & OVS_DP_F_UNALIGNED)
 478                 hlen = skb_zerocopy_headlen(skb);
 479         else
 480                 hlen = skb->len;
 481
 482         len = upcall_msg_size(upcall_info->userdata, hlen);
 483         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 484         if (!user_skb) {
 485                 err = -ENOMEM;
 486                 goto out;
 487         }
 488
 489         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 490                              0, upcall_info->cmd);
 491         upcall->dp_ifindex = dp_ifindex;
 492
 493         nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_KEY);
 494         err = ovs_nla_put_flow(upcall_info->key, upcall_info->key, user_skb);
 495         BUG_ON(err);
 496         nla_nest_end(user_skb, nla);
 497
 498         if (upcall_info->userdata)
 499                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 500                           nla_len(upcall_info->userdata),
 501                           nla_data(upcall_info->userdata));
 502
 503         /* Only reserve room for attribute header, packet data is added
 504          * in skb_zerocopy() */
 505         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 506                 err = -ENOBUFS;
 507                 goto out;
 508         }
 509         nla->nla_len = nla_attr_size(skb->len);
 510
 511         err = skb_zerocopy(user_skb, skb, skb->len, hlen);
 512         if (err)
 513                 goto out;
 514
 515         /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 516         if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 517                 size_t plen = NLA_ALIGN(user_skb->len) - user_skb->len;
 518
 519                 if (plen > 0)
 520                         memset(skb_put(user_skb, plen), 0, plen);
 521         }
 522
 523         ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 524
 525         err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 526 out:
 527         if (err)
 528                 skb_tx_error(skb);
 529         kfree_skb(nskb);
 530         return err;
 531 }
 532
 533 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 534 {
 535         struct ovs_header *ovs_header = info->userhdr;
 536         struct nlattr **a = info->attrs;
 537         struct sw_flow_actions *acts;
 538         struct sk_buff *packet;
 539         struct sw_flow *flow;
 540         struct datapath *dp;
 541         struct ethhdr *eth;
 542         struct vport *input_vport;
 543         int len;
 544         int err;
 545
 546         err = -EINVAL;
 547         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 548             !a[OVS_PACKET_ATTR_ACTIONS])
 549                 goto err;
 550
 551         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 552         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 553         err = -ENOMEM;
 554         if (!packet)
 555                 goto err;
 556         skb_reserve(packet, NET_IP_ALIGN);
 557
 558         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 559
 560         skb_reset_mac_header(packet);
 561         eth = eth_hdr(packet);
 562
 563         /* Normally, setting the skb 'protocol' field would be handled by a
 564          * call to eth_type_trans(), but it assumes there's a sending
 565          * device, which we may not have. */
 566         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 567                 packet->protocol = eth->h_proto;
 568         else
 569                 packet->protocol = htons(ETH_P_802_2);
 570
 571         /* Build an sw_flow for sending this packet. */
 572         flow = ovs_flow_alloc();
 573         err = PTR_ERR(flow);
 574         if (IS_ERR(flow))
 575                 goto err_kfree_skb;
 576
 577         err = ovs_flow_key_extract_userspace(a[OVS_PACKET_ATTR_KEY], packet,
 578                                              &flow->key);
 579         if (err)
 580                 goto err_flow_free;
 581
 582         acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_PACKET_ATTR_ACTIONS]));
 583         err = PTR_ERR(acts);
 584         if (IS_ERR(acts))
 585                 goto err_flow_free;
 586
 587         err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
 588                                    &flow->key, 0, &acts);
 589         rcu_assign_pointer(flow->sf_acts, acts);
 590         if (err)
 591                 goto err_flow_free;
 592
 593         OVS_CB(packet)->flow = flow;
 594         OVS_CB(packet)->pkt_key = &flow->key;
 595         packet->priority = flow->key.phy.priority;
 596         packet->mark = flow->key.phy.skb_mark;
 597
 598         rcu_read_lock();
 599         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 600         err = -ENODEV;
 601         if (!dp)
 602                 goto err_unlock;
 603
 604         input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 605         if (!input_vport)
 606                 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 607
 608         if (!input_vport)
 609                 goto err_unlock;
 610
 611         OVS_CB(packet)->input_vport = input_vport;
 612
 613         local_bh_disable();
 614         err = ovs_execute_actions(dp, packet, false);
 615         local_bh_enable();
 616         rcu_read_unlock();
 617
 618         ovs_flow_free(flow, false);
 619         return err;
 620
 621 err_unlock:
 622         rcu_read_unlock();
 623 err_flow_free:
 624         ovs_flow_free(flow, false);
 625 err_kfree_skb:
 626         kfree_skb(packet);
 627 err:
 628         return err;
 629 }
 630
 631 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 632         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 633         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 634         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 635 };
 636
 637 static struct genl_ops dp_packet_genl_ops[] = {
 638         { .cmd = OVS_PACKET_CMD_EXECUTE,
 639           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 640           .policy = packet_policy,
 641           .doit = ovs_packet_cmd_execute
 642         }
 643 };
 644
 645 static struct genl_family dp_packet_genl_family = {
 646         .id = GENL_ID_GENERATE,
 647         .hdrsize = sizeof(struct ovs_header),
 648         .name = OVS_PACKET_FAMILY,
 649         .version = OVS_PACKET_VERSION,
 650         .maxattr = OVS_PACKET_ATTR_MAX,
 651         .netnsok = true,
 652         .parallel_ops = true,
 653         .ops = dp_packet_genl_ops,
 654         .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
 655 };
 656
 657 static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats,
 658                          struct ovs_dp_megaflow_stats *mega_stats)
 659 {
 660         int i;
 661
 662         memset(mega_stats, 0, sizeof(*mega_stats));
 663
 664         stats->n_flows = ovs_flow_tbl_count(&dp->table);
 665         mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 666
 667         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 668
 669         for_each_possible_cpu(i) {
 670                 const struct dp_stats_percpu *percpu_stats;
 671                 struct dp_stats_percpu local_stats;
 672                 unsigned int start;
 673
 674                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 675
 676                 do {
 677                         start = u64_stats_fetch_begin_irq(&percpu_stats->sync);
 678                         local_stats = *percpu_stats;
 679                 } while (u64_stats_fetch_retry_irq(&percpu_stats->sync, start));
 680
 681                 stats->n_hit += local_stats.n_hit;
 682                 stats->n_missed += local_stats.n_missed;
 683                 stats->n_lost += local_stats.n_lost;
 684                 mega_stats->n_mask_hit += local_stats.n_mask_hit;
 685         }
 686 }
 687
 688 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts)
 689 {
 690         return NLMSG_ALIGN(sizeof(struct ovs_header))
 691                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_KEY */
 692                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_MASK */
 693                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 694                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 695                 + nla_total_size(8) /* OVS_FLOW_ATTR_USED */
 696                 + nla_total_size(acts->actions_len); /* OVS_FLOW_ATTR_ACTIONS */
 697 }
 698
 699 /* Called with ovs_mutex or RCU read lock. */
 700 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
 701                                   struct sk_buff *skb, u32 portid,
 702                                   u32 seq, u32 flags, u8 cmd)
 703 {
 704         const int skb_orig_len = skb->len;
 705         struct nlattr *start;
 706         struct ovs_flow_stats stats;
 707         __be16 tcp_flags;
 708         unsigned long used;
 709         struct ovs_header *ovs_header;
 710         struct nlattr *nla;
 711         int err;
 712
 713         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
 714         if (!ovs_header)
 715                 return -EMSGSIZE;
 716
 717         ovs_header->dp_ifindex = dp_ifindex;
 718
 719         /* Fill flow key. */
 720         nla = nla_nest_start(skb, OVS_FLOW_ATTR_KEY);
 721         if (!nla)
 722                 goto nla_put_failure;
 723
 724         err = ovs_nla_put_flow(&flow->unmasked_key, &flow->unmasked_key, skb);
 725         if (err)
 726                 goto error;
 727         nla_nest_end(skb, nla);
 728
 729         nla = nla_nest_start(skb, OVS_FLOW_ATTR_MASK);
 730         if (!nla)
 731                 goto nla_put_failure;
 732
 733         err = ovs_nla_put_flow(&flow->key, &flow->mask->key, skb);
 734         if (err)
 735                 goto error;
 736
 737         nla_nest_end(skb, nla);
 738
 739         ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 740
 741         if (used &&
 742             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
 743                 goto nla_put_failure;
 744
 745         if (stats.n_packets &&
 746             nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
 747                 goto nla_put_failure;
 748
 749         if ((u8)ntohs(tcp_flags) &&
 750              nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 751                 goto nla_put_failure;
 752
 753         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 754          * this is the first flow to be dumped into 'skb'.  This is unusual for
 755          * Netlink but individual action lists can be longer than
 756          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 757          * The userspace caller can always fetch the actions separately if it
 758          * really wants them.  (Most userspace callers in fact don't care.)
 759          *
 760          * This can only fail for dump operations because the skb is always
 761          * properly sized for single flows.
 762          */
 763         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 764         if (start) {
 765                 const struct sw_flow_actions *sf_acts;
 766
 767                 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 768                 err = ovs_nla_put_actions(sf_acts->actions,
 769                                           sf_acts->actions_len, skb);
 770
 771                 if (!err)
 772                         nla_nest_end(skb, start);
 773                 else {
 774                         if (skb_orig_len)
 775                                 goto error;
 776
 777                         nla_nest_cancel(skb, start);
 778                 }
 779         } else if (skb_orig_len)
 780                 goto nla_put_failure;
 781
 782         return genlmsg_end(skb, ovs_header);
 783
 784 nla_put_failure:
 785         err = -EMSGSIZE;
 786 error:
 787         genlmsg_cancel(skb, ovs_header);
 788         return err;
 789 }
 790
 791 /* May not be called with RCU read lock. */
 792 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 793                                                struct genl_info *info,
 794                                                bool always)
 795 {
 796         struct sk_buff *skb;
 797
 798         if (!always && !ovs_must_notify(info, &ovs_dp_flow_multicast_group))
 799                 return NULL;
 800
 801         skb = genlmsg_new_unicast(ovs_flow_cmd_msg_size(acts), info, GFP_KERNEL);
 802
 803         if (!skb)
 804                 return ERR_PTR(-ENOMEM);
 805
 806         return skb;
 807 }
 808
 809 /* Called with ovs_mutex. */
 810 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
 811                                                int dp_ifindex,
 812                                                struct genl_info *info, u8 cmd,
 813                                                bool always)
 814 {
 815         struct sk_buff *skb;
 816         int retval;
 817
 818         skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), info,
 819                                       always);
 820         if (!skb || IS_ERR(skb))
 821                 return skb;
 822
 823         retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
 824                                         info->snd_portid, info->snd_seq, 0,
 825                                         cmd);
 826         BUG_ON(retval < 0);
 827         return skb;
 828 }
 829
 830 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 831 {
 832         struct nlattr **a = info->attrs;
 833         struct ovs_header *ovs_header = info->userhdr;
 834         struct sw_flow *flow, *new_flow;
 835         struct sw_flow_mask mask;
 836         struct sk_buff *reply;
 837         struct datapath *dp;
 838         struct sw_flow_actions *acts;
 839         struct sw_flow_match match;
 840         int error;
 841
 842         /* Must have key and actions. */
 843         error = -EINVAL;
 844         if (!a[OVS_FLOW_ATTR_KEY])
 845                 goto error;
 846         if (!a[OVS_FLOW_ATTR_ACTIONS])
 847                 goto error;
 848
 849         /* Most of the time we need to allocate a new flow, do it before
 850          * locking. */
 851         new_flow = ovs_flow_alloc();
 852         if (IS_ERR(new_flow)) {
 853                 error = PTR_ERR(new_flow);
 854                 goto error;
 855         }
 856
 857         /* Extract key. */
 858         ovs_match_init(&match, &new_flow->unmasked_key, &mask);
 859         error = ovs_nla_get_match(&match,
 860                                   a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
 861         if (error)
 862                 goto err_kfree_flow;
 863
 864         ovs_flow_mask_key(&new_flow->key, &new_flow->unmasked_key, &mask);
 865
 866         /* Validate actions. */
 867         acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
 868         error = PTR_ERR(acts);
 869         if (IS_ERR(acts))
 870                 goto err_kfree_flow;
 871
 872         error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &new_flow->key,
 873                                      0, &acts);
 874         if (error) {
 875                 OVS_NLERR("Flow actions may not be safe on all matching packets.\n");
 876                 goto err_kfree_acts;
 877         }
 878
 879         reply = ovs_flow_cmd_alloc_info(acts, info, false);
 880         if (IS_ERR(reply)) {
 881                 error = PTR_ERR(reply);
 882                 goto err_kfree_acts;
 883         }
 884
 885         ovs_lock();
 886         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 887         if (unlikely(!dp)) {
 888                 error = -ENODEV;
 889                 goto err_unlock_ovs;
 890         }
 891         /* Check if this is a duplicate flow */
 892         flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->unmasked_key);
 893         if (likely(!flow)) {
 894                 rcu_assign_pointer(new_flow->sf_acts, acts);
 895
 896                 /* Put flow in bucket. */
 897                 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
 898                 if (unlikely(error)) {
 899                         acts = NULL;
 900                         goto err_unlock_ovs;
 901                 }
 902
 903                 if (unlikely(reply)) {
 904                         error = ovs_flow_cmd_fill_info(new_flow,
 905                                                        ovs_header->dp_ifindex,
 906                                                        reply, info->snd_portid,
 907                                                        info->snd_seq, 0,
 908                                                        OVS_FLOW_CMD_NEW);
 909                         BUG_ON(error < 0);
 910                 }
 911                 ovs_unlock();
 912         } else {
 913                 struct sw_flow_actions *old_acts;
 914
 915                 /* Bail out if we're not allowed to modify an existing flow.
 916                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 917                  * because Generic Netlink treats the latter as a dump
 918                  * request.  We also accept NLM_F_EXCL in case that bug ever
 919                  * gets fixed.
 920                  */
 921                 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
 922                                                          | NLM_F_EXCL))) {
 923                         error = -EEXIST;
 924                         goto err_unlock_ovs;
 925                 }
 926                 /* The unmasked key has to be the same for flow updates. */
 927                 if (unlikely(!ovs_flow_cmp_unmasked_key(flow, &match))) {
 928                         /* Look for any overlapping flow. */
 929                         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
 930                         if (!flow) {
 931                                 error = -ENOENT;
 932                                 goto err_unlock_ovs;
 933                         }
 934                 }
 935                 /* Update actions. */
 936                 old_acts = ovsl_dereference(flow->sf_acts);
 937                 rcu_assign_pointer(flow->sf_acts, acts);
 938
 939                 if (unlikely(reply)) {
 940                         error = ovs_flow_cmd_fill_info(flow,
 941                                                        ovs_header->dp_ifindex,
 942                                                        reply, info->snd_portid,
 943                                                        info->snd_seq, 0,
 944                                                        OVS_FLOW_CMD_NEW);
 945                         BUG_ON(error < 0);
 946                 }
 947                 ovs_unlock();
 948
 949                 ovs_nla_free_flow_actions(old_acts);
 950                 ovs_flow_free(new_flow, false);
 951         }
 952
 953         if (reply)
 954                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
 955         return 0;
 956
 957 err_unlock_ovs:
 958         ovs_unlock();
 959         kfree_skb(reply);
 960 err_kfree_acts:
 961         kfree(acts);
 962 err_kfree_flow:
 963         ovs_flow_free(new_flow, false);
 964 error:
 965         return error;
 966 }
 967
 968 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
 969 {
 970         struct nlattr **a = info->attrs;
 971         struct ovs_header *ovs_header = info->userhdr;
 972         struct sw_flow_key key, masked_key;
 973         struct sw_flow *flow;
 974         struct sw_flow_mask mask;
 975         struct sk_buff *reply = NULL;
 976         struct datapath *dp;
 977         struct sw_flow_actions *old_acts = NULL, *acts = NULL;
 978         struct sw_flow_match match;
 979         int error;
 980
 981         /* Extract key. */
 982         error = -EINVAL;
 983         if (!a[OVS_FLOW_ATTR_KEY])
 984                 goto error;
 985
 986         ovs_match_init(&match, &key, &mask);
 987         error = ovs_nla_get_match(&match,
 988                                   a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
 989         if (error)
 990                 goto error;
 991
 992         /* Validate actions. */
 993         if (a[OVS_FLOW_ATTR_ACTIONS]) {
 994                 acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
 995                 error = PTR_ERR(acts);
 996                 if (IS_ERR(acts))
 997                         goto error;
 998
 999                 ovs_flow_mask_key(&masked_key, &key, &mask);
1000                 error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS],
1001                                              &masked_key, 0, &acts);
1002                 if (error) {
1003                         OVS_NLERR("Flow actions may not be safe on all matching packets.\n");
1004                         goto err_kfree_acts;
1005                 }
1006         }
1007
1008         /* Can allocate before locking if have acts. */
1009         if (acts) {
1010                 reply = ovs_flow_cmd_alloc_info(acts, info, false);
1011                 if (IS_ERR(reply)) {
1012                         error = PTR_ERR(reply);
1013                         goto err_kfree_acts;
1014                 }
1015         }
1016
1017         ovs_lock();
1018         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1019         if (unlikely(!dp)) {
1020                 error = -ENODEV;
1021                 goto err_unlock_ovs;
1022         }
1023         /* Check that the flow exists. */
1024         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1025         if (unlikely(!flow)) {
1026                 error = -ENOENT;
1027                 goto err_unlock_ovs;
1028         }
1029
1030         /* Update actions, if present. */
1031         if (likely(acts)) {
1032                 old_acts = ovsl_dereference(flow->sf_acts);
1033                 rcu_assign_pointer(flow->sf_acts, acts);
1034
1035                 if (unlikely(reply)) {
1036                         error = ovs_flow_cmd_fill_info(flow,
1037                                                        ovs_header->dp_ifindex,
1038                                                        reply, info->snd_portid,
1039                                                        info->snd_seq, 0,
1040                                                        OVS_FLOW_CMD_NEW);
1041                         BUG_ON(error < 0);
1042                 }
1043         } else {
1044                 /* Could not alloc without acts before locking. */
1045                 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1046                                                 info, OVS_FLOW_CMD_NEW, false);
1047                 if (unlikely(IS_ERR(reply))) {
1048                         error = PTR_ERR(reply);
1049                         goto err_unlock_ovs;
1050                 }
1051         }
1052
1053         /* Clear stats. */
1054         if (a[OVS_FLOW_ATTR_CLEAR])
1055                 ovs_flow_stats_clear(flow);
1056         ovs_unlock();
1057
1058         if (reply)
1059                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1060         if (old_acts)
1061                 ovs_nla_free_flow_actions(old_acts);
1062         return 0;
1063
1064 err_unlock_ovs:
1065         ovs_unlock();
1066         kfree_skb(reply);
1067 err_kfree_acts:
1068         kfree(acts);
1069 error:
1070         return error;
1071 }
1072
1073 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1074 {
1075         struct nlattr **a = info->attrs;
1076         struct ovs_header *ovs_header = info->userhdr;
1077         struct sw_flow_key key;
1078         struct sk_buff *reply;
1079         struct sw_flow *flow;
1080         struct datapath *dp;
1081         struct sw_flow_match match;
1082         int err;
1083
1084         if (!a[OVS_FLOW_ATTR_KEY]) {
1085                 OVS_NLERR("Flow get message rejected, Key attribute missing.\n");
1086                 return -EINVAL;
1087         }
1088
1089         ovs_match_init(&match, &key, NULL);
1090         err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
1091         if (err)
1092                 return err;
1093
1094         ovs_lock();
1095         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1096         if (!dp) {
1097                 err = -ENODEV;
1098                 goto unlock;
1099         }
1100
1101         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1102         if (!flow) {
1103                 err = -ENOENT;
1104                 goto unlock;
1105         }
1106
1107         reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1108                                         OVS_FLOW_CMD_NEW, true);
1109         if (IS_ERR(reply)) {
1110                 err = PTR_ERR(reply);
1111                 goto unlock;
1112         }
1113
1114         ovs_unlock();
1115         return genlmsg_reply(reply, info);
1116 unlock:
1117         ovs_unlock();
1118         return err;
1119 }
1120
1121 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1122 {
1123         struct nlattr **a = info->attrs;
1124         struct ovs_header *ovs_header = info->userhdr;
1125         struct sw_flow_key key;
1126         struct sk_buff *reply;
1127         struct sw_flow *flow;
1128         struct datapath *dp;
1129         struct sw_flow_match match;
1130         int err;
1131
1132         if (likely(a[OVS_FLOW_ATTR_KEY])) {
1133                 ovs_match_init(&match, &key, NULL);
1134                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
1135                 if (unlikely(err))
1136                         return err;
1137         }
1138
1139         ovs_lock();
1140         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1141         if (unlikely(!dp)) {
1142                 err = -ENODEV;
1143                 goto unlock;
1144         }
1145         if (unlikely(!a[OVS_FLOW_ATTR_KEY])) {
1146                 err = ovs_flow_tbl_flush(&dp->table);
1147                 goto unlock;
1148         }
1149         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1150         if (unlikely(!flow)) {
1151                 err = -ENOENT;
1152                 goto unlock;
1153         }
1154
1155         ovs_flow_tbl_remove(&dp->table, flow);
1156         ovs_unlock();
1157
1158         reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *)flow->sf_acts,
1159                                         info, false);
1160
1161         if (likely(reply)) {
1162                 if (likely(!IS_ERR(reply))) {
1163                         rcu_read_lock(); /* Keep RCU checker happy. */
1164                         err = ovs_flow_cmd_fill_info(flow,
1165                                                      ovs_header->dp_ifindex,
1166                                                      reply, info->snd_portid,
1167                                                      info->snd_seq, 0,
1168                                                      OVS_FLOW_CMD_DEL);
1169                         rcu_read_unlock();
1170                         BUG_ON(err < 0);
1171                         ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1172                 } else {
1173                         genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1174                                      GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1175
1176                 }
1177         }
1178
1179         ovs_flow_free(flow, true);
1180         return 0;
1181 unlock:
1182         ovs_unlock();
1183         return err;
1184 }
1185
1186 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1187 {
1188         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1189         struct table_instance *ti;
1190         struct datapath *dp;
1191
1192         rcu_read_lock();
1193         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1194         if (!dp) {
1195                 rcu_read_unlock();
1196                 return -ENODEV;
1197         }
1198
1199         ti = rcu_dereference(dp->table.ti);
1200         for (;;) {
1201                 struct sw_flow *flow;
1202                 u32 bucket, obj;
1203
1204                 bucket = cb->args[0];
1205                 obj = cb->args[1];
1206                 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1207                 if (!flow)
1208                         break;
1209
1210                 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1211                                            NETLINK_CB(cb->skb).portid,
1212                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1213                                            OVS_FLOW_CMD_NEW) < 0)
1214                         break;
1215
1216                 cb->args[0] = bucket;
1217                 cb->args[1] = obj;
1218         }
1219         rcu_read_unlock();
1220         return skb->len;
1221 }
1222
1223 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1224         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1225         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1226         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1227 };
1228
1229 static struct genl_ops dp_flow_genl_ops[] = {
1230         { .cmd = OVS_FLOW_CMD_NEW,
1231           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1232           .policy = flow_policy,
1233           .doit = ovs_flow_cmd_new
1234         },
1235         { .cmd = OVS_FLOW_CMD_DEL,
1236           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1237           .policy = flow_policy,
1238           .doit = ovs_flow_cmd_del
1239         },
1240         { .cmd = OVS_FLOW_CMD_GET,
1241           .flags = 0,               /* OK for unprivileged users. */
1242           .policy = flow_policy,
1243           .doit = ovs_flow_cmd_get,
1244           .dumpit = ovs_flow_cmd_dump
1245         },
1246         { .cmd = OVS_FLOW_CMD_SET,
1247           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1248           .policy = flow_policy,
1249           .doit = ovs_flow_cmd_set,
1250         },
1251 };
1252
1253 static struct genl_family dp_flow_genl_family = {
1254         .id = GENL_ID_GENERATE,
1255         .hdrsize = sizeof(struct ovs_header),
1256         .name = OVS_FLOW_FAMILY,
1257         .version = OVS_FLOW_VERSION,
1258         .maxattr = OVS_FLOW_ATTR_MAX,
1259         .netnsok = true,
1260         .parallel_ops = true,
1261         .ops = dp_flow_genl_ops,
1262         .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1263         .mcgrps = &ovs_dp_flow_multicast_group,
1264         .n_mcgrps = 1,
1265 };
1266
1267 static size_t ovs_dp_cmd_msg_size(void)
1268 {
1269         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1270
1271         msgsize += nla_total_size(IFNAMSIZ);
1272         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1273         msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1274         msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1275
1276         return msgsize;
1277 }
1278
1279 /* Called with ovs_mutex or RCU read lock. */
1280 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1281                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1282 {
1283         struct ovs_header *ovs_header;
1284         struct ovs_dp_stats dp_stats;
1285         struct ovs_dp_megaflow_stats dp_megaflow_stats;
1286         int err;
1287
1288         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1289                                    flags, cmd);
1290         if (!ovs_header)
1291                 goto error;
1292
1293         ovs_header->dp_ifindex = get_dpifindex(dp);
1294
1295         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1296         if (err)
1297                 goto nla_put_failure;
1298
1299         get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1300         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1301                         &dp_stats))
1302                 goto nla_put_failure;
1303
1304         if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1305                         sizeof(struct ovs_dp_megaflow_stats),
1306                         &dp_megaflow_stats))
1307                 goto nla_put_failure;
1308
1309         if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1310                 goto nla_put_failure;
1311
1312         return genlmsg_end(skb, ovs_header);
1313
1314 nla_put_failure:
1315         genlmsg_cancel(skb, ovs_header);
1316 error:
1317         return -EMSGSIZE;
1318 }
1319
1320 static struct sk_buff *ovs_dp_cmd_alloc_info(struct genl_info *info)
1321 {
1322         return genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1323 }
1324
1325 /* Called with rcu_read_lock or ovs_mutex. */
1326 static struct datapath *lookup_datapath(struct net *net,
1327                                         struct ovs_header *ovs_header,
1328                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1329 {
1330         struct datapath *dp;
1331
1332         if (!a[OVS_DP_ATTR_NAME])
1333                 dp = get_dp(net, ovs_header->dp_ifindex);
1334         else {
1335                 struct vport *vport;
1336
1337                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1338                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1339         }
1340         return dp ? dp : ERR_PTR(-ENODEV);
1341 }
1342
1343 static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1344 {
1345         struct datapath *dp;
1346
1347         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1348         if (IS_ERR(dp))
1349                 return;
1350
1351         WARN(dp->user_features, "Dropping previously announced user features\n");
1352         dp->user_features = 0;
1353 }
1354
1355 static void ovs_dp_change(struct datapath *dp, struct nlattr **a)
1356 {
1357         if (a[OVS_DP_ATTR_USER_FEATURES])
1358                 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1359 }
1360
1361 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1362 {
1363         struct nlattr **a = info->attrs;
1364         struct vport_parms parms;
1365         struct sk_buff *reply;
1366         struct datapath *dp;
1367         struct vport *vport;
1368         struct ovs_net *ovs_net;
1369         int err, i;
1370
1371         err = -EINVAL;
1372         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1373                 goto err;
1374
1375         reply = ovs_dp_cmd_alloc_info(info);
1376         if (!reply)
1377                 return -ENOMEM;
1378
1379         err = -ENOMEM;
1380         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1381         if (dp == NULL)
1382                 goto err_free_reply;
1383
1384         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1385
1386         /* Allocate table. */
1387         err = ovs_flow_tbl_init(&dp->table);
1388         if (err)
1389                 goto err_free_dp;
1390
1391         dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1392         if (!dp->stats_percpu) {
1393                 err = -ENOMEM;
1394                 goto err_destroy_table;
1395         }
1396
1397         for_each_possible_cpu(i) {
1398                 struct dp_stats_percpu *dpath_stats;
1399                 dpath_stats = per_cpu_ptr(dp->stats_percpu, i);
1400                 u64_stats_init(&dpath_stats->sync);
1401         }
1402
1403         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1404                             GFP_KERNEL);
1405         if (!dp->ports) {
1406                 err = -ENOMEM;
1407                 goto err_destroy_percpu;
1408         }
1409
1410         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1411                 INIT_HLIST_HEAD(&dp->ports[i]);
1412
1413         /* Set up our datapath device. */
1414         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1415         parms.type = OVS_VPORT_TYPE_INTERNAL;
1416         parms.options = NULL;
1417         parms.dp = dp;
1418         parms.port_no = OVSP_LOCAL;
1419         parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1420
1421         ovs_dp_change(dp, a);
1422
1423         /* So far only local changes have been made, now need the lock. */
1424         ovs_lock();
1425
1426         vport = new_vport(&parms);
1427         if (IS_ERR(vport)) {
1428                 err = PTR_ERR(vport);
1429                 if (err == -EBUSY)
1430                         err = -EEXIST;
1431
1432                 if (err == -EEXIST) {
1433                         /* An outdated user space instance that does not understand
1434                          * the concept of user_features has attempted to create a new
1435                          * datapath and is likely to reuse it. Drop all user features.
1436                          */
1437                         if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1438                                 ovs_dp_reset_user_features(skb, info);
1439                 }
1440
1441                 goto err_destroy_ports_array;
1442         }
1443
1444         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1445                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1446         BUG_ON(err < 0);
1447
1448         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1449         list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1450
1451         ovs_unlock();
1452
1453         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1454         return 0;
1455
1456 err_destroy_ports_array:
1457         ovs_unlock();
1458         kfree(dp->ports);
1459 err_destroy_percpu:
1460         free_percpu(dp->stats_percpu);
1461 err_destroy_table:
1462         ovs_flow_tbl_destroy(&dp->table);
1463 err_free_dp:
1464         release_net(ovs_dp_get_net(dp));
1465         kfree(dp);
1466 err_free_reply:
1467         kfree_skb(reply);
1468 err:
1469         return err;
1470 }
1471
1472 /* Called with ovs_mutex. */
1473 static void __dp_destroy(struct datapath *dp)
1474 {
1475         int i;
1476
1477         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1478                 struct vport *vport;
1479                 struct hlist_node *n;
1480
1481                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1482                         if (vport->port_no != OVSP_LOCAL)
1483                                 ovs_dp_detach_port(vport);
1484         }
1485
1486         list_del_rcu(&dp->list_node);
1487
1488         /* OVSP_LOCAL is datapath internal port. We need to make sure that
1489          * all ports in datapath are destroyed first before freeing datapath.
1490          */
1491         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1492
1493         /* RCU destroy the flow table */
1494         call_rcu(&dp->rcu, destroy_dp_rcu);
1495 }
1496
1497 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1498 {
1499         struct sk_buff *reply;
1500         struct datapath *dp;
1501         int err;
1502
1503         reply = ovs_dp_cmd_alloc_info(info);
1504         if (!reply)
1505                 return -ENOMEM;
1506
1507         ovs_lock();
1508         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1509         err = PTR_ERR(dp);
1510         if (IS_ERR(dp))
1511                 goto err_unlock_free;
1512
1513         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1514                                    info->snd_seq, 0, OVS_DP_CMD_DEL);
1515         BUG_ON(err < 0);
1516
1517         __dp_destroy(dp);
1518
1519         ovs_unlock();
1520         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1521         return 0;
1522
1523 err_unlock_free:
1524         ovs_unlock();
1525         kfree_skb(reply);
1526         return err;
1527 }
1528
1529 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1530 {
1531         struct sk_buff *reply;
1532         struct datapath *dp;
1533         int err;
1534
1535         reply = ovs_dp_cmd_alloc_info(info);
1536         if (!reply)
1537                 return -ENOMEM;
1538
1539         ovs_lock();
1540         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1541         err = PTR_ERR(dp);
1542         if (IS_ERR(dp))
1543                 goto err_unlock_free;
1544
1545         ovs_dp_change(dp, info->attrs);
1546
1547         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1548                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1549         BUG_ON(err < 0);
1550
1551         ovs_unlock();
1552         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1553         return 0;
1554
1555 err_unlock_free:
1556         ovs_unlock();
1557         kfree_skb(reply);
1558         return err;
1559 }
1560
1561 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1562 {
1563         struct sk_buff *reply;
1564         struct datapath *dp;
1565         int err;
1566
1567         reply = ovs_dp_cmd_alloc_info(info);
1568         if (!reply)
1569                 return -ENOMEM;
1570
1571         rcu_read_lock();
1572         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1573         if (IS_ERR(dp)) {
1574                 err = PTR_ERR(dp);
1575                 goto err_unlock_free;
1576         }
1577         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1578                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1579         BUG_ON(err < 0);
1580         rcu_read_unlock();
1581
1582         return genlmsg_reply(reply, info);
1583
1584 err_unlock_free:
1585         rcu_read_unlock();
1586         kfree_skb(reply);
1587         return err;
1588 }
1589
1590 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1591 {
1592         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1593         struct datapath *dp;
1594         int skip = cb->args[0];
1595         int i = 0;
1596
1597         rcu_read_lock();
1598         list_for_each_entry_rcu(dp, &ovs_net->dps, list_node) {
1599                 if (i >= skip &&
1600                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1601                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1602                                          OVS_DP_CMD_NEW) < 0)
1603                         break;
1604                 i++;
1605         }
1606         rcu_read_unlock();
1607
1608         cb->args[0] = i;
1609
1610         return skb->len;
1611 }
1612
1613 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1614         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1615         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1616         [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1617 };
1618
1619 static struct genl_ops dp_datapath_genl_ops[] = {
1620         { .cmd = OVS_DP_CMD_NEW,
1621           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1622           .policy = datapath_policy,
1623           .doit = ovs_dp_cmd_new
1624         },
1625         { .cmd = OVS_DP_CMD_DEL,
1626           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1627           .policy = datapath_policy,
1628           .doit = ovs_dp_cmd_del
1629         },
1630         { .cmd = OVS_DP_CMD_GET,
1631           .flags = 0,               /* OK for unprivileged users. */
1632           .policy = datapath_policy,
1633           .doit = ovs_dp_cmd_get,
1634           .dumpit = ovs_dp_cmd_dump
1635         },
1636         { .cmd = OVS_DP_CMD_SET,
1637           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1638           .policy = datapath_policy,
1639           .doit = ovs_dp_cmd_set,
1640         },
1641 };
1642
1643 static struct genl_family dp_datapath_genl_family = {
1644         .id = GENL_ID_GENERATE,
1645         .hdrsize = sizeof(struct ovs_header),
1646         .name = OVS_DATAPATH_FAMILY,
1647         .version = OVS_DATAPATH_VERSION,
1648         .maxattr = OVS_DP_ATTR_MAX,
1649         .netnsok = true,
1650         .parallel_ops = true,
1651         .ops = dp_datapath_genl_ops,
1652         .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1653         .mcgrps = &ovs_dp_datapath_multicast_group,
1654         .n_mcgrps = 1,
1655 };
1656
1657 /* Called with ovs_mutex or RCU read lock. */
1658 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1659                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1660 {
1661         struct ovs_header *ovs_header;
1662         struct ovs_vport_stats vport_stats;
1663         int err;
1664
1665         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1666                                  flags, cmd);
1667         if (!ovs_header)
1668                 return -EMSGSIZE;
1669
1670         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1671
1672         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1673             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1674             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)))
1675                 goto nla_put_failure;
1676
1677         ovs_vport_get_stats(vport, &vport_stats);
1678         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1679                     &vport_stats))
1680                 goto nla_put_failure;
1681
1682         if (ovs_vport_get_upcall_portids(vport, skb))
1683                 goto nla_put_failure;
1684
1685         err = ovs_vport_get_options(vport, skb);
1686         if (err == -EMSGSIZE)
1687                 goto error;
1688
1689         return genlmsg_end(skb, ovs_header);
1690
1691 nla_put_failure:
1692         err = -EMSGSIZE;
1693 error:
1694         genlmsg_cancel(skb, ovs_header);
1695         return err;
1696 }
1697
1698 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1699 {
1700         return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1701 }
1702
1703 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1704 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1705                                          u32 seq, u8 cmd)
1706 {
1707         struct sk_buff *skb;
1708         int retval;
1709
1710         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1711         if (!skb)
1712                 return ERR_PTR(-ENOMEM);
1713
1714         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1715         BUG_ON(retval < 0);
1716
1717         return skb;
1718 }
1719
1720 /* Called with ovs_mutex or RCU read lock. */
1721 static struct vport *lookup_vport(struct net *net,
1722                                   struct ovs_header *ovs_header,
1723                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1724 {
1725         struct datapath *dp;
1726         struct vport *vport;
1727
1728         if (a[OVS_VPORT_ATTR_NAME]) {
1729                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1730                 if (!vport)
1731                         return ERR_PTR(-ENODEV);
1732                 if (ovs_header->dp_ifindex &&
1733                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1734                         return ERR_PTR(-ENODEV);
1735                 return vport;
1736         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1737                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1738
1739                 if (port_no >= DP_MAX_PORTS)
1740                         return ERR_PTR(-EFBIG);
1741
1742                 dp = get_dp(net, ovs_header->dp_ifindex);
1743                 if (!dp)
1744                         return ERR_PTR(-ENODEV);
1745
1746                 vport = ovs_vport_ovsl_rcu(dp, port_no);
1747                 if (!vport)
1748                         return ERR_PTR(-ENODEV);
1749                 return vport;
1750         } else
1751                 return ERR_PTR(-EINVAL);
1752 }
1753
1754 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1755 {
1756         struct nlattr **a = info->attrs;
1757         struct ovs_header *ovs_header = info->userhdr;
1758         struct vport_parms parms;
1759         struct sk_buff *reply;
1760         struct vport *vport;
1761         struct datapath *dp;
1762         u32 port_no;
1763         int err;
1764
1765         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1766             !a[OVS_VPORT_ATTR_UPCALL_PID])
1767                 return -EINVAL;
1768
1769         port_no = a[OVS_VPORT_ATTR_PORT_NO]
1770                 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
1771         if (port_no >= DP_MAX_PORTS)
1772                 return -EFBIG;
1773
1774         reply = ovs_vport_cmd_alloc_info();
1775         if (!reply)
1776                 return -ENOMEM;
1777
1778         ovs_lock();
1779         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1780         err = -ENODEV;
1781         if (!dp)
1782                 goto exit_unlock_free;
1783
1784         if (port_no) {
1785                 vport = ovs_vport_ovsl(dp, port_no);
1786                 err = -EBUSY;
1787                 if (vport)
1788                         goto exit_unlock_free;
1789         } else {
1790                 for (port_no = 1; ; port_no++) {
1791                         if (port_no >= DP_MAX_PORTS) {
1792                                 err = -EFBIG;
1793                                 goto exit_unlock_free;
1794                         }
1795                         vport = ovs_vport_ovsl(dp, port_no);
1796                         if (!vport)
1797                                 break;
1798                 }
1799         }
1800
1801         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1802         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1803         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1804         parms.dp = dp;
1805         parms.port_no = port_no;
1806         parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
1807
1808         vport = new_vport(&parms);
1809         err = PTR_ERR(vport);
1810         if (IS_ERR(vport))
1811                 goto exit_unlock_free;
1812
1813         err = 0;
1814         if (a[OVS_VPORT_ATTR_STATS])
1815                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
1816
1817         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1818                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1819         BUG_ON(err < 0);
1820         ovs_unlock();
1821
1822         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1823         return 0;
1824
1825 exit_unlock_free:
1826         ovs_unlock();
1827         kfree_skb(reply);
1828         return err;
1829 }
1830
1831 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
1832 {
1833         struct nlattr **a = info->attrs;
1834         struct sk_buff *reply;
1835         struct vport *vport;
1836         int err;
1837
1838         reply = ovs_vport_cmd_alloc_info();
1839         if (!reply)
1840                 return -ENOMEM;
1841
1842         ovs_lock();
1843         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1844         err = PTR_ERR(vport);
1845         if (IS_ERR(vport))
1846                 goto exit_unlock_free;
1847
1848         if (a[OVS_VPORT_ATTR_TYPE] &&
1849             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1850                 err = -EINVAL;
1851                 goto exit_unlock_free;
1852         }
1853
1854         if (a[OVS_VPORT_ATTR_OPTIONS]) {
1855                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
1856                 if (err)
1857                         goto exit_unlock_free;
1858         }
1859
1860         if (a[OVS_VPORT_ATTR_STATS])
1861                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
1862
1863
1864         if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
1865                 err = ovs_vport_set_upcall_portids(vport,
1866                                                    a[OVS_VPORT_ATTR_UPCALL_PID]);
1867                 if (err)
1868                         goto exit_unlock_free;
1869         }
1870
1871         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1872                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1873         BUG_ON(err < 0);
1874         ovs_unlock();
1875
1876         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1877         return 0;
1878
1879 exit_unlock_free:
1880         ovs_unlock();
1881         kfree_skb(reply);
1882         return err;
1883 }
1884
1885 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
1886 {
1887         struct nlattr **a = info->attrs;
1888         struct sk_buff *reply;
1889         struct vport *vport;
1890         int err;
1891
1892         reply = ovs_vport_cmd_alloc_info();
1893         if (!reply)
1894                 return -ENOMEM;
1895
1896         ovs_lock();
1897         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1898         err = PTR_ERR(vport);
1899         if (IS_ERR(vport))
1900                 goto exit_unlock_free;
1901
1902         if (vport->port_no == OVSP_LOCAL) {
1903                 err = -EINVAL;
1904                 goto exit_unlock_free;
1905         }
1906
1907         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1908                                       info->snd_seq, 0, OVS_VPORT_CMD_DEL);
1909         BUG_ON(err < 0);
1910         ovs_dp_detach_port(vport);
1911         ovs_unlock();
1912
1913         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1914         return 0;
1915
1916 exit_unlock_free:
1917         ovs_unlock();
1918         kfree_skb(reply);
1919         return err;
1920 }
1921
1922 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
1923 {
1924         struct nlattr **a = info->attrs;
1925         struct ovs_header *ovs_header = info->userhdr;
1926         struct sk_buff *reply;
1927         struct vport *vport;
1928         int err;
1929
1930         reply = ovs_vport_cmd_alloc_info();
1931         if (!reply)
1932                 return -ENOMEM;
1933
1934         rcu_read_lock();
1935         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
1936         err = PTR_ERR(vport);
1937         if (IS_ERR(vport))
1938                 goto exit_unlock_free;
1939         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1940                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1941         BUG_ON(err < 0);
1942         rcu_read_unlock();
1943
1944         return genlmsg_reply(reply, info);
1945
1946 exit_unlock_free:
1947         rcu_read_unlock();
1948         kfree_skb(reply);
1949         return err;
1950 }
1951
1952 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1953 {
1954         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1955         struct datapath *dp;
1956         int bucket = cb->args[0], skip = cb->args[1];
1957         int i, j = 0;
1958
1959         rcu_read_lock();
1960         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1961         if (!dp) {
1962                 rcu_read_unlock();
1963                 return -ENODEV;
1964         }
1965         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
1966                 struct vport *vport;
1967
1968                 j = 0;
1969                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
1970                         if (j >= skip &&
1971                             ovs_vport_cmd_fill_info(vport, skb,
1972                                                     NETLINK_CB(cb->skb).portid,
1973                                                     cb->nlh->nlmsg_seq,
1974                                                     NLM_F_MULTI,
1975                                                     OVS_VPORT_CMD_NEW) < 0)
1976                                 goto out;
1977
1978                         j++;
1979                 }
1980                 skip = 0;
1981         }
1982 out:
1983         rcu_read_unlock();
1984
1985         cb->args[0] = i;
1986         cb->args[1] = j;
1987
1988         return skb->len;
1989 }
1990
1991 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
1992         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1993         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
1994         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
1995         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
1996         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1997         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
1998 };
1999
2000 static struct genl_ops dp_vport_genl_ops[] = {
2001         { .cmd = OVS_VPORT_CMD_NEW,
2002           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2003           .policy = vport_policy,
2004           .doit = ovs_vport_cmd_new
2005         },
2006         { .cmd = OVS_VPORT_CMD_DEL,
2007           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2008           .policy = vport_policy,
2009           .doit = ovs_vport_cmd_del
2010         },
2011         { .cmd = OVS_VPORT_CMD_GET,
2012           .flags = 0,               /* OK for unprivileged users. */
2013           .policy = vport_policy,
2014           .doit = ovs_vport_cmd_get,
2015           .dumpit = ovs_vport_cmd_dump
2016         },
2017         { .cmd = OVS_VPORT_CMD_SET,
2018           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2019           .policy = vport_policy,
2020           .doit = ovs_vport_cmd_set,
2021         },
2022 };
2023
2024 struct genl_family dp_vport_genl_family = {
2025         .id = GENL_ID_GENERATE,
2026         .hdrsize = sizeof(struct ovs_header),
2027         .name = OVS_VPORT_FAMILY,
2028         .version = OVS_VPORT_VERSION,
2029         .maxattr = OVS_VPORT_ATTR_MAX,
2030         .netnsok = true,
2031         .parallel_ops = true,
2032         .ops = dp_vport_genl_ops,
2033         .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2034         .mcgrps = &ovs_dp_vport_multicast_group,
2035         .n_mcgrps = 1,
2036 };
2037
2038 static struct genl_family *dp_genl_families[] = {
2039         &dp_datapath_genl_family,
2040         &dp_vport_genl_family,
2041         &dp_flow_genl_family,
2042         &dp_packet_genl_family,
2043 };
2044
2045 static void dp_unregister_genl(int n_families)
2046 {
2047         int i;
2048
2049         for (i = 0; i < n_families; i++)
2050                 genl_unregister_family(dp_genl_families[i]);
2051 }
2052
2053 static int dp_register_genl(void)
2054 {
2055         int err;
2056         int i;
2057
2058         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2059
2060                 err = genl_register_family(dp_genl_families[i]);
2061                 if (err)
2062                         goto error;
2063         }
2064
2065         return 0;
2066
2067 error:
2068         dp_unregister_genl(i);
2069         return err;
2070 }
2071
2072 static int __net_init ovs_init_net(struct net *net)
2073 {
2074         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2075
2076         INIT_LIST_HEAD(&ovs_net->dps);
2077         INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2078         return 0;
2079 }
2080
2081 static void __net_exit ovs_exit_net(struct net *net)
2082 {
2083         struct datapath *dp, *dp_next;
2084         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2085
2086         ovs_lock();
2087         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2088                 __dp_destroy(dp);
2089         ovs_unlock();
2090
2091         cancel_work_sync(&ovs_net->dp_notify_work);
2092 }
2093
2094 static struct pernet_operations ovs_net_ops = {
2095         .init = ovs_init_net,
2096         .exit = ovs_exit_net,
2097         .id   = &ovs_net_id,
2098         .size = sizeof(struct ovs_net),
2099 };
2100
2101 DEFINE_COMPAT_PNET_REG_FUNC(device);
2102
2103 static int __init dp_init(void)
2104 {
2105         int err;
2106
2107         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2108
2109         pr_info("Open vSwitch switching datapath %s, built "__DATE__" "__TIME__"\n",
2110                 VERSION);
2111
2112         err = ovs_flow_init();
2113         if (err)
2114                 goto error;
2115
2116         err = ovs_vport_init();
2117         if (err)
2118                 goto error_flow_exit;
2119
2120         err = register_pernet_device(&ovs_net_ops);
2121         if (err)
2122                 goto error_vport_exit;
2123
2124         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2125         if (err)
2126                 goto error_netns_exit;
2127
2128         err = dp_register_genl();
2129         if (err < 0)
2130                 goto error_unreg_notifier;
2131
2132         return 0;
2133
2134 error_unreg_notifier:
2135         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2136 error_netns_exit:
2137         unregister_pernet_device(&ovs_net_ops);
2138 error_vport_exit:
2139         ovs_vport_exit();
2140 error_flow_exit:
2141         ovs_flow_exit();
2142 error:
2143         return err;
2144 }
2145
2146 static void dp_cleanup(void)
2147 {
2148         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2149         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2150         unregister_pernet_device(&ovs_net_ops);
2151         rcu_barrier();
2152         ovs_vport_exit();
2153         ovs_flow_exit();
2154 }
2155
2156 module_init(dp_init);
2157 module_exit(dp_cleanup);
2158
2159 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2160 MODULE_LICENSE("GPL");
2161 MODULE_VERSION(VERSION);