datapath/datapath.c

   1 /*
   2  * Copyright (c) 2007-2014 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/version.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/wait.h>
  42 #include <asm/div64.h>
  43 #include <linux/highmem.h>
  44 #include <linux/netfilter_bridge.h>
  45 #include <linux/netfilter_ipv4.h>
  46 #include <linux/inetdevice.h>
  47 #include <linux/list.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <linux/genetlink.h>
  52 #include <net/genetlink.h>
  53 #include <net/genetlink.h>
  54 #include <net/net_namespace.h>
  55 #include <net/netns/generic.h>
  56
  57 #include "datapath.h"
  58 #include "flow.h"
  59 #include "flow_table.h"
  60 #include "flow_netlink.h"
  61 #include "vlan.h"
  62 #include "vport-internal_dev.h"
  63 #include "vport-netdev.h"
  64
  65 int ovs_net_id __read_mostly;
  66
  67 static struct genl_family dp_packet_genl_family;
  68 static struct genl_family dp_flow_genl_family;
  69 static struct genl_family dp_datapath_genl_family;
  70
  71 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
  72         .name = OVS_FLOW_MCGROUP
  73 };
  74
  75 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  76         .name = OVS_DATAPATH_MCGROUP
  77 };
  78
  79 struct genl_multicast_group ovs_dp_vport_multicast_group = {
  80         .name = OVS_VPORT_MCGROUP
  81 };
  82
  83 /* Check if need to build a reply message.
  84  * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
  85 static bool ovs_must_notify(struct genl_info *info,
  86                             const struct genl_multicast_group *grp)
  87 {
  88         return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  89                 netlink_has_listeners(genl_info_net(info)->genl_sock, GROUP_ID(grp));
  90 }
  91
  92 static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
  93                        struct sk_buff *skb, struct genl_info *info)
  94 {
  95         genl_notify(family, skb, genl_info_net(info),
  96                     info->snd_portid, GROUP_ID(grp), info->nlhdr, GFP_KERNEL);
  97 }
  98
  99 /**
 100  * DOC: Locking:
 101  *
 102  * All writes e.g. Writes to device state (add/remove datapath, port, set
 103  * operations on vports, etc.), Writes to other state (flow table
 104  * modifications, set miscellaneous datapath parameters, etc.) are protected
 105  * by ovs_lock.
 106  *
 107  * Reads are protected by RCU.
 108  *
 109  * There are a few special cases (mostly stats) that have their own
 110  * synchronization but they nest under all of above and don't interact with
 111  * each other.
 112  *
 113  * The RTNL lock nests inside ovs_mutex.
 114  */
 115
 116 static DEFINE_MUTEX(ovs_mutex);
 117
 118 void ovs_lock(void)
 119 {
 120         mutex_lock(&ovs_mutex);
 121 }
 122
 123 void ovs_unlock(void)
 124 {
 125         mutex_unlock(&ovs_mutex);
 126 }
 127
 128 #ifdef CONFIG_LOCKDEP
 129 int lockdep_ovsl_is_held(void)
 130 {
 131         if (debug_locks)
 132                 return lockdep_is_held(&ovs_mutex);
 133         else
 134                 return 1;
 135 }
 136 #endif
 137
 138 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 139                              const struct dp_upcall_info *);
 140 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 141                                   const struct dp_upcall_info *);
 142
 143 /* Must be called with rcu_read_lock. */
 144 static struct datapath *get_dp_rcu(struct net *net, int dp_ifindex)
 145 {
 146         struct net_device *dev = dev_get_by_index_rcu(net, dp_ifindex);
 147
 148         if (dev) {
 149                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 150                 if (vport)
 151                         return vport->dp;
 152         }
 153
 154         return NULL;
 155 }
 156
 157 /* The caller must hold either ovs_mutex or rcu_read_lock to keep the
 158  * returned dp pointer valid. */
 159 static inline struct datapath *get_dp(struct net *net, int dp_ifindex)
 160 {
 161         struct datapath *dp;
 162
 163         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_ovsl_is_held());
 164         rcu_read_lock();
 165         dp = get_dp_rcu(net, dp_ifindex);
 166         rcu_read_unlock();
 167
 168         return dp;
 169 }
 170
 171 /* Must be called with rcu_read_lock or ovs_mutex. */
 172 const char *ovs_dp_name(const struct datapath *dp)
 173 {
 174         struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 175         return vport->ops->get_name(vport);
 176 }
 177
 178 static int get_dpifindex(struct datapath *dp)
 179 {
 180         struct vport *local;
 181         int ifindex;
 182
 183         rcu_read_lock();
 184
 185         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 186         if (local)
 187                 ifindex = netdev_vport_priv(local)->dev->ifindex;
 188         else
 189                 ifindex = 0;
 190
 191         rcu_read_unlock();
 192
 193         return ifindex;
 194 }
 195
 196 static void destroy_dp_rcu(struct rcu_head *rcu)
 197 {
 198         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 199
 200         ovs_flow_tbl_destroy(&dp->table);
 201         free_percpu(dp->stats_percpu);
 202         release_net(ovs_dp_get_net(dp));
 203         kfree(dp->ports);
 204         kfree(dp);
 205 }
 206
 207 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 208                                             u16 port_no)
 209 {
 210         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 211 }
 212
 213 /* Called with ovs_mutex or RCU read lock. */
 214 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 215 {
 216         struct vport *vport;
 217         struct hlist_head *head;
 218
 219         head = vport_hash_bucket(dp, port_no);
 220         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 221                 if (vport->port_no == port_no)
 222                         return vport;
 223         }
 224         return NULL;
 225 }
 226
 227 /* Called with ovs_mutex. */
 228 static struct vport *new_vport(const struct vport_parms *parms)
 229 {
 230         struct vport *vport;
 231
 232         vport = ovs_vport_add(parms);
 233         if (!IS_ERR(vport)) {
 234                 struct datapath *dp = parms->dp;
 235                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 236
 237                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 238         }
 239         return vport;
 240 }
 241
 242 void ovs_dp_detach_port(struct vport *p)
 243 {
 244         ASSERT_OVSL();
 245
 246         /* First drop references to device. */
 247         hlist_del_rcu(&p->dp_hash_node);
 248
 249         /* Then destroy it. */
 250         ovs_vport_del(p);
 251 }
 252
 253 void ovs_dp_process_packet_with_key(struct sk_buff *skb,
 254                                     struct sw_flow_key *pkt_key,
 255                                     bool recirc)
 256 {
 257         const struct vport *p = OVS_CB(skb)->input_vport;
 258         struct datapath *dp = p->dp;
 259         struct sw_flow *flow;
 260         struct dp_stats_percpu *stats;
 261         u64 *stats_counter;
 262         u32 n_mask_hit;
 263
 264         stats = this_cpu_ptr(dp->stats_percpu);
 265         OVS_CB(skb)->pkt_key = pkt_key;
 266
 267         /* Look up flow. */
 268         flow = ovs_flow_tbl_lookup_stats(&dp->table, pkt_key, skb_get_hash(skb),
 269                                          &n_mask_hit);
 270         if (unlikely(!flow)) {
 271                 struct dp_upcall_info upcall;
 272
 273                 upcall.cmd = OVS_PACKET_CMD_MISS;
 274                 upcall.userdata = NULL;
 275                 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 276                 ovs_dp_upcall(dp, skb, &upcall);
 277                 consume_skb(skb);
 278                 stats_counter = &stats->n_missed;
 279                 goto out;
 280         }
 281
 282         OVS_CB(skb)->flow = flow;
 283
 284         ovs_flow_stats_update(OVS_CB(skb)->flow, pkt_key->tp.flags, skb);
 285         ovs_execute_actions(dp, skb, recirc);
 286         stats_counter = &stats->n_hit;
 287
 288 out:
 289         /* Update datapath statistics. */
 290         u64_stats_update_begin(&stats->sync);
 291         (*stats_counter)++;
 292         stats->n_mask_hit += n_mask_hit;
 293         u64_stats_update_end(&stats->sync);
 294 }
 295
 296 /* Must be called with rcu_read_lock. */
 297 void ovs_dp_process_received_packet(struct sk_buff *skb)
 298 {
 299         int error;
 300         struct sw_flow_key key;
 301
 302         /* Extract flow from 'skb' into 'key'. */
 303         error = ovs_flow_key_extract(skb, &key);
 304         if (unlikely(error)) {
 305                 kfree_skb(skb);
 306                 return;
 307         }
 308
 309         ovs_dp_process_packet_with_key(skb, &key, false);
 310 }
 311
 312 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 313                   const struct dp_upcall_info *upcall_info)
 314 {
 315         struct dp_stats_percpu *stats;
 316         int err;
 317
 318         BUG_ON(!OVS_CB(skb)->pkt_key);
 319
 320         if (upcall_info->portid == 0) {
 321                 err = -ENOTCONN;
 322                 goto err;
 323         }
 324
 325         if (!skb_is_gso(skb))
 326                 err = queue_userspace_packet(dp, skb, upcall_info);
 327         else
 328                 err = queue_gso_packets(dp, skb, upcall_info);
 329         if (err)
 330                 goto err;
 331
 332         return 0;
 333
 334 err:
 335         stats = this_cpu_ptr(dp->stats_percpu);
 336
 337         u64_stats_update_begin(&stats->sync);
 338         stats->n_lost++;
 339         u64_stats_update_end(&stats->sync);
 340
 341         return err;
 342 }
 343
 344 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 345                              const struct dp_upcall_info *upcall_info)
 346 {
 347         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 348         struct sw_flow_key later_key;
 349         struct sk_buff *segs, *nskb;
 350         int err;
 351
 352         segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 353         if (IS_ERR(segs))
 354                 return PTR_ERR(segs);
 355
 356         if (gso_type & SKB_GSO_UDP) {
 357                 /* The initial flow key extracted by ovs_flow_key_extract()
 358                  * in this case is for a first fragment, so we need to
 359                  * properly mark later fragments.
 360                  */
 361                 later_key = *OVS_CB(skb)->pkt_key;
 362                 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 363         }
 364
 365         /* Queue all of the segments. */
 366         skb = segs;
 367         do {
 368                 if (gso_type & SKB_GSO_UDP && skb != segs)
 369                         OVS_CB(skb)->pkt_key = &later_key;
 370
 371                 err = queue_userspace_packet(dp, skb, upcall_info);
 372                 if (err)
 373                         break;
 374
 375         } while ((skb = skb->next));
 376
 377         /* Free all of the segments. */
 378         skb = segs;
 379         do {
 380                 nskb = skb->next;
 381                 if (err)
 382                         kfree_skb(skb);
 383                 else
 384                         consume_skb(skb);
 385         } while ((skb = nskb));
 386         return err;
 387 }
 388
 389 static size_t key_attr_size(void)
 390 {
 391         /* Whenever adding new OVS_KEY_ FIELDS, we should consider
 392          * updating this function.  */
 393         BUILD_BUG_ON(OVS_KEY_ATTR_TUNNEL_INFO != 22);
 394
 395         return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 396                 + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
 397                   + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
 398                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
 399                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
 400                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
 401                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
 402                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
 403                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
 404                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_OAM */
 405                   + nla_total_size(256) /* OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS */
 406                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 407                 + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 408                 + nla_total_size(4)   /* OVS_KEY_ATTR_DP_HASH */
 409                 + nla_total_size(4)   /* OVS_KEY_ATTR_RECIRC_ID */
 410                 + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
 411                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 412                 + nla_total_size(4)   /* OVS_KEY_ATTR_8021Q */
 413                 + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
 414                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 415                 + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
 416                 + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
 417                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 418 }
 419
 420 static size_t upcall_msg_size(const struct nlattr *userdata,
 421                               unsigned int hdrlen)
 422 {
 423         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 424                 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 425                 + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 426
 427         /* OVS_PACKET_ATTR_USERDATA */
 428         if (userdata)
 429                 size += NLA_ALIGN(userdata->nla_len);
 430
 431         return size;
 432 }
 433
 434 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 435                                   const struct dp_upcall_info *upcall_info)
 436 {
 437         struct ovs_header *upcall;
 438         struct sk_buff *nskb = NULL;
 439         struct sk_buff *user_skb; /* to be queued to userspace */
 440         struct sw_flow_key *pkt_key = OVS_CB(skb)->pkt_key;
 441         struct nlattr *nla;
 442         struct genl_info info = {
 443 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
 444                 .dst_sk = ovs_dp_get_net(dp)->genl_sock,
 445 #endif
 446                 .snd_portid = upcall_info->portid,
 447         };
 448         size_t len;
 449         unsigned int hlen;
 450         int err, dp_ifindex;
 451
 452         dp_ifindex = get_dpifindex(dp);
 453         if (!dp_ifindex)
 454                 return -ENODEV;
 455
 456         if (vlan_tx_tag_present(skb)) {
 457                 nskb = skb_clone(skb, GFP_ATOMIC);
 458                 if (!nskb)
 459                         return -ENOMEM;
 460
 461                 nskb = __vlan_put_tag(nskb, nskb->vlan_proto, vlan_tx_tag_get(nskb));
 462                 if (!nskb)
 463                         return -ENOMEM;
 464
 465                 vlan_set_tci(nskb, 0);
 466
 467                 skb = nskb;
 468         }
 469
 470         if (nla_attr_size(skb->len) > USHRT_MAX) {
 471                 err = -EFBIG;
 472                 goto out;
 473         }
 474
 475         /* Complete checksum if needed */
 476         if (skb->ip_summed == CHECKSUM_PARTIAL &&
 477             (err = skb_checksum_help(skb)))
 478                 goto out;
 479
 480         /* Older versions of OVS user space enforce alignment of the last
 481          * Netlink attribute to NLA_ALIGNTO which would require extensive
 482          * padding logic. Only perform zerocopy if padding is not required.
 483          */
 484         if (dp->user_features & OVS_DP_F_UNALIGNED)
 485                 hlen = skb_zerocopy_headlen(skb);
 486         else
 487                 hlen = skb->len;
 488
 489         len = upcall_msg_size(upcall_info->userdata, hlen);
 490         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 491         if (!user_skb) {
 492                 err = -ENOMEM;
 493                 goto out;
 494         }
 495
 496         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 497                              0, upcall_info->cmd);
 498         upcall->dp_ifindex = dp_ifindex;
 499
 500         nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_KEY);
 501         err = ovs_nla_put_flow(dp, pkt_key, pkt_key, user_skb);
 502         BUG_ON(err);
 503         nla_nest_end(user_skb, nla);
 504
 505         if (upcall_info->userdata)
 506                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 507                           nla_len(upcall_info->userdata),
 508                           nla_data(upcall_info->userdata));
 509
 510         /* Only reserve room for attribute header, packet data is added
 511          * in skb_zerocopy() */
 512         if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 513                 err = -ENOBUFS;
 514                 goto out;
 515         }
 516         nla->nla_len = nla_attr_size(skb->len);
 517
 518         err = skb_zerocopy(user_skb, skb, skb->len, hlen);
 519         if (err)
 520                 goto out;
 521
 522         /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 523         if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 524                 size_t plen = NLA_ALIGN(user_skb->len) - user_skb->len;
 525
 526                 if (plen > 0)
 527                         memset(skb_put(user_skb, plen), 0, plen);
 528         }
 529
 530         ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 531
 532         err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 533 out:
 534         if (err)
 535                 skb_tx_error(skb);
 536         kfree_skb(nskb);
 537         return err;
 538 }
 539
 540 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 541 {
 542         struct ovs_header *ovs_header = info->userhdr;
 543         struct nlattr **a = info->attrs;
 544         struct sw_flow_actions *acts;
 545         struct sk_buff *packet;
 546         struct sw_flow *flow;
 547         struct datapath *dp;
 548         struct ethhdr *eth;
 549         struct vport *input_vport;
 550         int len;
 551         int err;
 552
 553         err = -EINVAL;
 554         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 555             !a[OVS_PACKET_ATTR_ACTIONS])
 556                 goto err;
 557
 558         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 559         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 560         err = -ENOMEM;
 561         if (!packet)
 562                 goto err;
 563         skb_reserve(packet, NET_IP_ALIGN);
 564
 565         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 566
 567         skb_reset_mac_header(packet);
 568         eth = eth_hdr(packet);
 569
 570         /* Normally, setting the skb 'protocol' field would be handled by a
 571          * call to eth_type_trans(), but it assumes there's a sending
 572          * device, which we may not have. */
 573         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 574                 packet->protocol = eth->h_proto;
 575         else
 576                 packet->protocol = htons(ETH_P_802_2);
 577
 578         /* Build an sw_flow for sending this packet. */
 579         flow = ovs_flow_alloc();
 580         err = PTR_ERR(flow);
 581         if (IS_ERR(flow))
 582                 goto err_kfree_skb;
 583
 584         err = ovs_flow_key_extract_userspace(a[OVS_PACKET_ATTR_KEY], packet,
 585                                              &flow->key);
 586         if (err)
 587                 goto err_flow_free;
 588
 589         acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_PACKET_ATTR_ACTIONS]));
 590         err = PTR_ERR(acts);
 591         if (IS_ERR(acts))
 592                 goto err_flow_free;
 593
 594         err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
 595                                    &flow->key, &acts);
 596         rcu_assign_pointer(flow->sf_acts, acts);
 597         if (err)
 598                 goto err_flow_free;
 599
 600         OVS_CB(packet)->flow = flow;
 601         OVS_CB(packet)->pkt_key = &flow->key;
 602         packet->priority = flow->key.phy.priority;
 603         packet->mark = flow->key.phy.skb_mark;
 604
 605         rcu_read_lock();
 606         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
 607         err = -ENODEV;
 608         if (!dp)
 609                 goto err_unlock;
 610
 611         input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 612         if (!input_vport)
 613                 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 614
 615         if (!input_vport)
 616                 goto err_unlock;
 617
 618         OVS_CB(packet)->input_vport = input_vport;
 619
 620         local_bh_disable();
 621         err = ovs_execute_actions(dp, packet, false);
 622         local_bh_enable();
 623         rcu_read_unlock();
 624
 625         ovs_flow_free(flow, false);
 626         return err;
 627
 628 err_unlock:
 629         rcu_read_unlock();
 630 err_flow_free:
 631         ovs_flow_free(flow, false);
 632 err_kfree_skb:
 633         kfree_skb(packet);
 634 err:
 635         return err;
 636 }
 637
 638 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 639         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 640         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 641         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 642 };
 643
 644 static struct genl_ops dp_packet_genl_ops[] = {
 645         { .cmd = OVS_PACKET_CMD_EXECUTE,
 646           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 647           .policy = packet_policy,
 648           .doit = ovs_packet_cmd_execute
 649         }
 650 };
 651
 652 static struct genl_family dp_packet_genl_family = {
 653         .id = GENL_ID_GENERATE,
 654         .hdrsize = sizeof(struct ovs_header),
 655         .name = OVS_PACKET_FAMILY,
 656         .version = OVS_PACKET_VERSION,
 657         .maxattr = OVS_PACKET_ATTR_MAX,
 658         .netnsok = true,
 659         .parallel_ops = true,
 660         .ops = dp_packet_genl_ops,
 661         .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
 662 };
 663
 664 static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats,
 665                          struct ovs_dp_megaflow_stats *mega_stats)
 666 {
 667         int i;
 668
 669         memset(mega_stats, 0, sizeof(*mega_stats));
 670
 671         stats->n_flows = ovs_flow_tbl_count(&dp->table);
 672         mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 673
 674         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 675
 676         for_each_possible_cpu(i) {
 677                 const struct dp_stats_percpu *percpu_stats;
 678                 struct dp_stats_percpu local_stats;
 679                 unsigned int start;
 680
 681                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 682
 683                 do {
 684                         start = u64_stats_fetch_begin_irq(&percpu_stats->sync);
 685                         local_stats = *percpu_stats;
 686                 } while (u64_stats_fetch_retry_irq(&percpu_stats->sync, start));
 687
 688                 stats->n_hit += local_stats.n_hit;
 689                 stats->n_missed += local_stats.n_missed;
 690                 stats->n_lost += local_stats.n_lost;
 691                 mega_stats->n_mask_hit += local_stats.n_mask_hit;
 692         }
 693 }
 694
 695 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts)
 696 {
 697         return NLMSG_ALIGN(sizeof(struct ovs_header))
 698                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_KEY */
 699                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_MASK */
 700                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 701                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 702                 + nla_total_size(8) /* OVS_FLOW_ATTR_USED */
 703                 + nla_total_size(acts->actions_len); /* OVS_FLOW_ATTR_ACTIONS */
 704 }
 705
 706 /* Called with ovs_mutex or RCU read lock. */
 707 static int ovs_flow_cmd_fill_match(struct datapath *dp,
 708                                    const struct sw_flow *flow,
 709                                    struct sk_buff *skb)
 710 {
 711         struct nlattr *nla;
 712         int err;
 713
 714         /* Fill flow key. */
 715         nla = nla_nest_start(skb, OVS_FLOW_ATTR_KEY);
 716         if (!nla)
 717                 return -EMSGSIZE;
 718
 719         err = ovs_nla_put_flow(dp, &flow->unmasked_key,
 720                                &flow->unmasked_key, skb);
 721         if (err)
 722                 return err;
 723         nla_nest_end(skb, nla);
 724
 725         /* Fill flow mask. */
 726         nla = nla_nest_start(skb, OVS_FLOW_ATTR_MASK);
 727         if (!nla)
 728                 return -EMSGSIZE;
 729
 730         err = ovs_nla_put_flow(dp, &flow->key, &flow->mask->key, skb);
 731         if (err)
 732                 return err;
 733         nla_nest_end(skb, nla);
 734
 735         return 0;
 736 }
 737
 738 /* Called with ovs_mutex or RCU read lock. */
 739 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
 740                                    struct sk_buff *skb)
 741 {
 742         struct ovs_flow_stats stats;
 743         __be16 tcp_flags;
 744         unsigned long used;
 745
 746         ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 747
 748         if (used &&
 749             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
 750                 return -EMSGSIZE;
 751
 752         if (stats.n_packets &&
 753             nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
 754                 return -EMSGSIZE;
 755
 756         if ((u8)ntohs(tcp_flags) &&
 757              nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 758                 return -EMSGSIZE;
 759
 760         return 0;
 761 }
 762
 763 /* Called with ovs_mutex or RCU read lock. */
 764 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
 765                                      struct sk_buff *skb, int skb_orig_len)
 766 {
 767         struct nlattr *start;
 768         int err;
 769
 770         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 771          * this is the first flow to be dumped into 'skb'.  This is unusual for
 772          * Netlink but individual action lists can be longer than
 773          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 774          * The userspace caller can always fetch the actions separately if it
 775          * really wants them.  (Most userspace callers in fact don't care.)
 776          *
 777          * This can only fail for dump operations because the skb is always
 778          * properly sized for single flows.
 779          */
 780         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 781         if (start) {
 782                 const struct sw_flow_actions *sf_acts;
 783
 784                 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 785                 err = ovs_nla_put_actions(sf_acts->actions,
 786                                           sf_acts->actions_len, skb);
 787
 788                 if (!err)
 789                         nla_nest_end(skb, start);
 790                 else {
 791                         if (skb_orig_len)
 792                                 return err;
 793
 794                         nla_nest_cancel(skb, start);
 795                 }
 796         } else if (skb_orig_len) {
 797                 return -EMSGSIZE;
 798         }
 799
 800         return 0;
 801 }
 802
 803 /* Called with ovs_mutex or RCU read lock. */
 804 static int ovs_flow_cmd_fill_info(struct datapath *dp,
 805                                   const struct sw_flow *flow, int dp_ifindex,
 806                                   struct sk_buff *skb, u32 portid,
 807                                   u32 seq, u32 flags, u8 cmd)
 808 {
 809         const int skb_orig_len = skb->len;
 810         struct ovs_header *ovs_header;
 811         int err;
 812
 813         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
 814         if (!ovs_header)
 815                 return -EMSGSIZE;
 816         ovs_header->dp_ifindex = dp_ifindex;
 817
 818         err = ovs_flow_cmd_fill_match(dp, flow, skb);
 819         if (err)
 820                 goto error;
 821
 822         err = ovs_flow_cmd_fill_stats(flow, skb);
 823         if (err)
 824                 goto error;
 825
 826         err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
 827         if (err)
 828                 goto error;
 829
 830         return genlmsg_end(skb, ovs_header);
 831
 832 error:
 833         genlmsg_cancel(skb, ovs_header);
 834         return err;
 835 }
 836
 837 /* May not be called with RCU read lock. */
 838 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 839                                                struct genl_info *info,
 840                                                bool always)
 841 {
 842         struct sk_buff *skb;
 843
 844         if (!always && !ovs_must_notify(info, &ovs_dp_flow_multicast_group))
 845                 return NULL;
 846
 847         skb = genlmsg_new_unicast(ovs_flow_cmd_msg_size(acts), info, GFP_KERNEL);
 848
 849         if (!skb)
 850                 return ERR_PTR(-ENOMEM);
 851
 852         return skb;
 853 }
 854
 855 /* Called with ovs_mutex. */
 856 static struct sk_buff *ovs_flow_cmd_build_info(struct datapath *dp,
 857                                                const struct sw_flow *flow,
 858                                                int dp_ifindex,
 859                                                struct genl_info *info, u8 cmd,
 860                                                bool always)
 861 {
 862         struct sk_buff *skb;
 863         int retval;
 864
 865         skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), info,
 866                                       always);
 867         if (!skb || IS_ERR(skb))
 868                 return skb;
 869
 870         retval = ovs_flow_cmd_fill_info(dp, flow, dp_ifindex, skb,
 871                                         info->snd_portid, info->snd_seq, 0,
 872                                         cmd);
 873         BUG_ON(retval < 0);
 874         return skb;
 875 }
 876
 877 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 878 {
 879         struct nlattr **a = info->attrs;
 880         struct ovs_header *ovs_header = info->userhdr;
 881         struct sw_flow *flow, *new_flow;
 882         struct sw_flow_mask mask;
 883         struct sk_buff *reply;
 884         struct datapath *dp;
 885         struct sw_flow_actions *acts;
 886         struct sw_flow_match match;
 887         int error;
 888
 889         /* Must have key and actions. */
 890         error = -EINVAL;
 891         if (!a[OVS_FLOW_ATTR_KEY]) {
 892                 OVS_NLERR("Flow key attribute not present in new flow.\n");
 893                 goto error;
 894         }
 895         if (!a[OVS_FLOW_ATTR_ACTIONS]) {
 896                 OVS_NLERR("Flow actions attribute not present in new flow.\n");
 897                 goto error;
 898         }
 899
 900         /* Most of the time we need to allocate a new flow, do it before
 901          * locking. */
 902         new_flow = ovs_flow_alloc();
 903         if (IS_ERR(new_flow)) {
 904                 error = PTR_ERR(new_flow);
 905                 goto error;
 906         }
 907
 908         /* Extract key. */
 909         ovs_match_init(&match, &new_flow->unmasked_key, &mask);
 910         error = ovs_nla_get_match(&match,
 911                                   a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
 912         if (error)
 913                 goto err_kfree_flow;
 914
 915         ovs_flow_mask_key(&new_flow->key, &new_flow->unmasked_key, &mask);
 916
 917         /* Validate actions. */
 918         acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
 919         error = PTR_ERR(acts);
 920         if (IS_ERR(acts))
 921                 goto err_kfree_flow;
 922
 923         error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &new_flow->key,
 924                                      &acts);
 925         if (error) {
 926                 OVS_NLERR("Flow actions may not be safe on all matching packets.\n");
 927                 goto err_kfree_acts;
 928         }
 929
 930         reply = ovs_flow_cmd_alloc_info(acts, info, false);
 931         if (IS_ERR(reply)) {
 932                 error = PTR_ERR(reply);
 933                 goto err_kfree_acts;
 934         }
 935
 936         ovs_lock();
 937         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 938         if (unlikely(!dp)) {
 939                 error = -ENODEV;
 940                 goto err_unlock_ovs;
 941         }
 942         /* Check if this is a duplicate flow */
 943         flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->unmasked_key);
 944         if (likely(!flow)) {
 945                 rcu_assign_pointer(new_flow->sf_acts, acts);
 946
 947                 /* Put flow in bucket. */
 948                 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
 949                 if (unlikely(error)) {
 950                         acts = NULL;
 951                         goto err_unlock_ovs;
 952                 }
 953
 954                 if (unlikely(reply)) {
 955                         error = ovs_flow_cmd_fill_info(dp, new_flow,
 956                                                        ovs_header->dp_ifindex,
 957                                                        reply, info->snd_portid,
 958                                                        info->snd_seq, 0,
 959                                                        OVS_FLOW_CMD_NEW);
 960                         BUG_ON(error < 0);
 961                 }
 962                 ovs_unlock();
 963         } else {
 964                 struct sw_flow_actions *old_acts;
 965
 966                 /* Bail out if we're not allowed to modify an existing flow.
 967                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 968                  * because Generic Netlink treats the latter as a dump
 969                  * request.  We also accept NLM_F_EXCL in case that bug ever
 970                  * gets fixed.
 971                  */
 972                 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
 973                                                          | NLM_F_EXCL))) {
 974                         error = -EEXIST;
 975                         goto err_unlock_ovs;
 976                 }
 977                 /* The unmasked key has to be the same for flow updates. */
 978                 if (unlikely(!ovs_flow_cmp_unmasked_key(flow, &match))) {
 979                         /* Look for any overlapping flow. */
 980                         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
 981                         if (!flow) {
 982                                 error = -ENOENT;
 983                                 goto err_unlock_ovs;
 984                         }
 985                 }
 986                 /* Update actions. */
 987                 old_acts = ovsl_dereference(flow->sf_acts);
 988                 rcu_assign_pointer(flow->sf_acts, acts);
 989
 990                 if (unlikely(reply)) {
 991                         error = ovs_flow_cmd_fill_info(dp, flow,
 992                                                        ovs_header->dp_ifindex,
 993                                                        reply, info->snd_portid,
 994                                                        info->snd_seq, 0,
 995                                                        OVS_FLOW_CMD_NEW);
 996                         BUG_ON(error < 0);
 997                 }
 998                 ovs_unlock();
 999
1000                 ovs_nla_free_flow_actions(old_acts);
1001                 ovs_flow_free(new_flow, false);
1002         }
1003
1004         if (reply)
1005                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1006         return 0;
1007
1008 err_unlock_ovs:
1009         ovs_unlock();
1010         kfree_skb(reply);
1011 err_kfree_acts:
1012         kfree(acts);
1013 err_kfree_flow:
1014         ovs_flow_free(new_flow, false);
1015 error:
1016         return error;
1017 }
1018
1019 static struct sw_flow_actions *get_flow_actions(const struct nlattr *a,
1020                                                 const struct sw_flow_key *key,
1021                                                 const struct sw_flow_mask *mask)
1022 {
1023         struct sw_flow_actions *acts;
1024         struct sw_flow_key masked_key;
1025         int error;
1026
1027         acts = ovs_nla_alloc_flow_actions(nla_len(a));
1028         if (IS_ERR(acts))
1029                 return acts;
1030
1031         ovs_flow_mask_key(&masked_key, key, mask);
1032         error = ovs_nla_copy_actions(a, &masked_key, &acts);
1033         if (error) {
1034                 OVS_NLERR("Flow actions may not be safe on all matching packets.\n");
1035                 kfree(acts);
1036                 return ERR_PTR(error);
1037         }
1038
1039         return acts;
1040 }
1041
1042 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1043 {
1044         struct nlattr **a = info->attrs;
1045         struct ovs_header *ovs_header = info->userhdr;
1046         struct sw_flow_key key;
1047         struct sw_flow *flow;
1048         struct sw_flow_mask mask;
1049         struct sk_buff *reply = NULL;
1050         struct datapath *dp;
1051         struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1052         struct sw_flow_match match;
1053         int error;
1054
1055         /* Extract key. */
1056         error = -EINVAL;
1057         if (!a[OVS_FLOW_ATTR_KEY]) {
1058                 OVS_NLERR("Flow key attribute not present in set flow.\n");
1059                 goto error;
1060         }
1061
1062         ovs_match_init(&match, &key, &mask);
1063         error = ovs_nla_get_match(&match,
1064                                   a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
1065         if (error)
1066                 goto error;
1067
1068         /* Validate actions. */
1069         if (a[OVS_FLOW_ATTR_ACTIONS]) {
1070                 acts = get_flow_actions(a[OVS_FLOW_ATTR_ACTIONS], &key, &mask);
1071                 if (IS_ERR(acts)) {
1072                         error = PTR_ERR(acts);
1073                         goto error;
1074                 }
1075         }
1076
1077         /* Can allocate before locking if have acts. */
1078         if (acts) {
1079                 reply = ovs_flow_cmd_alloc_info(acts, info, false);
1080                 if (IS_ERR(reply)) {
1081                         error = PTR_ERR(reply);
1082                         goto err_kfree_acts;
1083                 }
1084         }
1085
1086         ovs_lock();
1087         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1088         if (unlikely(!dp)) {
1089                 error = -ENODEV;
1090                 goto err_unlock_ovs;
1091         }
1092         /* Check that the flow exists. */
1093         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1094         if (unlikely(!flow)) {
1095                 error = -ENOENT;
1096                 goto err_unlock_ovs;
1097         }
1098
1099         /* Update actions, if present. */
1100         if (likely(acts)) {
1101                 old_acts = ovsl_dereference(flow->sf_acts);
1102                 rcu_assign_pointer(flow->sf_acts, acts);
1103
1104                 if (unlikely(reply)) {
1105                         error = ovs_flow_cmd_fill_info(dp, flow,
1106                                                        ovs_header->dp_ifindex,
1107                                                        reply, info->snd_portid,
1108                                                        info->snd_seq, 0,
1109                                                        OVS_FLOW_CMD_NEW);
1110                         BUG_ON(error < 0);
1111                 }
1112         } else {
1113                 /* Could not alloc without acts before locking. */
1114                 reply = ovs_flow_cmd_build_info(dp, flow,
1115                                                 ovs_header->dp_ifindex,
1116                                                 info, OVS_FLOW_CMD_NEW, false);
1117                 if (unlikely(IS_ERR(reply))) {
1118                         error = PTR_ERR(reply);
1119                         goto err_unlock_ovs;
1120                 }
1121         }
1122
1123         /* Clear stats. */
1124         if (a[OVS_FLOW_ATTR_CLEAR])
1125                 ovs_flow_stats_clear(flow);
1126         ovs_unlock();
1127
1128         if (reply)
1129                 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1130         if (old_acts)
1131                 ovs_nla_free_flow_actions(old_acts);
1132         return 0;
1133
1134 err_unlock_ovs:
1135         ovs_unlock();
1136         kfree_skb(reply);
1137 err_kfree_acts:
1138         kfree(acts);
1139 error:
1140         return error;
1141 }
1142
1143 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1144 {
1145         struct nlattr **a = info->attrs;
1146         struct ovs_header *ovs_header = info->userhdr;
1147         struct sw_flow_key key;
1148         struct sk_buff *reply;
1149         struct sw_flow *flow;
1150         struct datapath *dp;
1151         struct sw_flow_match match;
1152         int err;
1153
1154         if (!a[OVS_FLOW_ATTR_KEY]) {
1155                 OVS_NLERR("Flow get message rejected, Key attribute missing.\n");
1156                 return -EINVAL;
1157         }
1158
1159         ovs_match_init(&match, &key, NULL);
1160         err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
1161         if (err)
1162                 return err;
1163
1164         ovs_lock();
1165         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1166         if (!dp) {
1167                 err = -ENODEV;
1168                 goto unlock;
1169         }
1170
1171         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1172         if (!flow) {
1173                 err = -ENOENT;
1174                 goto unlock;
1175         }
1176
1177         reply = ovs_flow_cmd_build_info(dp, flow, ovs_header->dp_ifindex, info,
1178                                         OVS_FLOW_CMD_NEW, true);
1179         if (IS_ERR(reply)) {
1180                 err = PTR_ERR(reply);
1181                 goto unlock;
1182         }
1183
1184         ovs_unlock();
1185         return genlmsg_reply(reply, info);
1186 unlock:
1187         ovs_unlock();
1188         return err;
1189 }
1190
1191 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1192 {
1193         struct nlattr **a = info->attrs;
1194         struct ovs_header *ovs_header = info->userhdr;
1195         struct sw_flow_key key;
1196         struct sk_buff *reply;
1197         struct sw_flow *flow;
1198         struct datapath *dp;
1199         struct sw_flow_match match;
1200         int err;
1201
1202         if (likely(a[OVS_FLOW_ATTR_KEY])) {
1203                 ovs_match_init(&match, &key, NULL);
1204                 err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
1205                 if (unlikely(err))
1206                         return err;
1207         }
1208
1209         ovs_lock();
1210         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1211         if (unlikely(!dp)) {
1212                 err = -ENODEV;
1213                 goto unlock;
1214         }
1215         if (unlikely(!a[OVS_FLOW_ATTR_KEY])) {
1216                 err = ovs_flow_tbl_flush(&dp->table);
1217                 goto unlock;
1218         }
1219         flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1220         if (unlikely(!flow)) {
1221                 err = -ENOENT;
1222                 goto unlock;
1223         }
1224
1225         ovs_flow_tbl_remove(&dp->table, flow);
1226         ovs_unlock();
1227
1228         reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *)flow->sf_acts,
1229                                         info, false);
1230
1231         if (likely(reply)) {
1232                 if (likely(!IS_ERR(reply))) {
1233                         rcu_read_lock(); /* Keep RCU checker happy. */
1234                         err = ovs_flow_cmd_fill_info(dp, flow,
1235                                                      ovs_header->dp_ifindex,
1236                                                      reply, info->snd_portid,
1237                                                      info->snd_seq, 0,
1238                                                      OVS_FLOW_CMD_DEL);
1239                         rcu_read_unlock();
1240                         BUG_ON(err < 0);
1241                         ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
1242                 } else {
1243                         genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1244                                      GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1245
1246                 }
1247         }
1248
1249         ovs_flow_free(flow, true);
1250         return 0;
1251 unlock:
1252         ovs_unlock();
1253         return err;
1254 }
1255
1256 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1257 {
1258         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1259         struct table_instance *ti;
1260         struct datapath *dp;
1261
1262         rcu_read_lock();
1263         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1264         if (!dp) {
1265                 rcu_read_unlock();
1266                 return -ENODEV;
1267         }
1268
1269         ti = rcu_dereference(dp->table.ti);
1270         for (;;) {
1271                 struct sw_flow *flow;
1272                 u32 bucket, obj;
1273
1274                 bucket = cb->args[0];
1275                 obj = cb->args[1];
1276                 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1277                 if (!flow)
1278                         break;
1279
1280                 if (ovs_flow_cmd_fill_info(dp, flow, ovs_header->dp_ifindex, skb,
1281                                            NETLINK_CB(cb->skb).portid,
1282                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1283                                            OVS_FLOW_CMD_NEW) < 0)
1284                         break;
1285
1286                 cb->args[0] = bucket;
1287                 cb->args[1] = obj;
1288         }
1289         rcu_read_unlock();
1290         return skb->len;
1291 }
1292
1293 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1294         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1295         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1296         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1297 };
1298
1299 static struct genl_ops dp_flow_genl_ops[] = {
1300         { .cmd = OVS_FLOW_CMD_NEW,
1301           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1302           .policy = flow_policy,
1303           .doit = ovs_flow_cmd_new
1304         },
1305         { .cmd = OVS_FLOW_CMD_DEL,
1306           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1307           .policy = flow_policy,
1308           .doit = ovs_flow_cmd_del
1309         },
1310         { .cmd = OVS_FLOW_CMD_GET,
1311           .flags = 0,               /* OK for unprivileged users. */
1312           .policy = flow_policy,
1313           .doit = ovs_flow_cmd_get,
1314           .dumpit = ovs_flow_cmd_dump
1315         },
1316         { .cmd = OVS_FLOW_CMD_SET,
1317           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1318           .policy = flow_policy,
1319           .doit = ovs_flow_cmd_set,
1320         },
1321 };
1322
1323 static struct genl_family dp_flow_genl_family = {
1324         .id = GENL_ID_GENERATE,
1325         .hdrsize = sizeof(struct ovs_header),
1326         .name = OVS_FLOW_FAMILY,
1327         .version = OVS_FLOW_VERSION,
1328         .maxattr = OVS_FLOW_ATTR_MAX,
1329         .netnsok = true,
1330         .parallel_ops = true,
1331         .ops = dp_flow_genl_ops,
1332         .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1333         .mcgrps = &ovs_dp_flow_multicast_group,
1334         .n_mcgrps = 1,
1335 };
1336
1337 static size_t ovs_dp_cmd_msg_size(void)
1338 {
1339         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1340
1341         msgsize += nla_total_size(IFNAMSIZ);
1342         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1343         msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1344         msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1345
1346         return msgsize;
1347 }
1348
1349 /* Called with ovs_mutex or RCU read lock. */
1350 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1351                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1352 {
1353         struct ovs_header *ovs_header;
1354         struct ovs_dp_stats dp_stats;
1355         struct ovs_dp_megaflow_stats dp_megaflow_stats;
1356         int err;
1357
1358         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1359                                    flags, cmd);
1360         if (!ovs_header)
1361                 goto error;
1362
1363         ovs_header->dp_ifindex = get_dpifindex(dp);
1364
1365         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1366         if (err)
1367                 goto nla_put_failure;
1368
1369         get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1370         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1371                         &dp_stats))
1372                 goto nla_put_failure;
1373
1374         if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1375                         sizeof(struct ovs_dp_megaflow_stats),
1376                         &dp_megaflow_stats))
1377                 goto nla_put_failure;
1378
1379         if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1380                 goto nla_put_failure;
1381
1382         return genlmsg_end(skb, ovs_header);
1383
1384 nla_put_failure:
1385         genlmsg_cancel(skb, ovs_header);
1386 error:
1387         return -EMSGSIZE;
1388 }
1389
1390 static struct sk_buff *ovs_dp_cmd_alloc_info(struct genl_info *info)
1391 {
1392         return genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1393 }
1394
1395 /* Called with rcu_read_lock or ovs_mutex. */
1396 static struct datapath *lookup_datapath(struct net *net,
1397                                         struct ovs_header *ovs_header,
1398                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1399 {
1400         struct datapath *dp;
1401
1402         if (!a[OVS_DP_ATTR_NAME])
1403                 dp = get_dp(net, ovs_header->dp_ifindex);
1404         else {
1405                 struct vport *vport;
1406
1407                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1408                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1409         }
1410         return dp ? dp : ERR_PTR(-ENODEV);
1411 }
1412
1413 static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1414 {
1415         struct datapath *dp;
1416
1417         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1418         if (IS_ERR(dp))
1419                 return;
1420
1421         WARN(dp->user_features, "Dropping previously announced user features\n");
1422         dp->user_features = 0;
1423 }
1424
1425 static void ovs_dp_change(struct datapath *dp, struct nlattr **a)
1426 {
1427         if (a[OVS_DP_ATTR_USER_FEATURES])
1428                 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1429 }
1430
1431 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1432 {
1433         struct nlattr **a = info->attrs;
1434         struct vport_parms parms;
1435         struct sk_buff *reply;
1436         struct datapath *dp;
1437         struct vport *vport;
1438         struct ovs_net *ovs_net;
1439         int err, i;
1440
1441         err = -EINVAL;
1442         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1443                 goto err;
1444
1445         reply = ovs_dp_cmd_alloc_info(info);
1446         if (!reply)
1447                 return -ENOMEM;
1448
1449         err = -ENOMEM;
1450         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1451         if (dp == NULL)
1452                 goto err_free_reply;
1453
1454         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1455
1456         /* Allocate table. */
1457         err = ovs_flow_tbl_init(&dp->table);
1458         if (err)
1459                 goto err_free_dp;
1460
1461         dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1462         if (!dp->stats_percpu) {
1463                 err = -ENOMEM;
1464                 goto err_destroy_table;
1465         }
1466
1467         for_each_possible_cpu(i) {
1468                 struct dp_stats_percpu *dpath_stats;
1469                 dpath_stats = per_cpu_ptr(dp->stats_percpu, i);
1470                 u64_stats_init(&dpath_stats->sync);
1471         }
1472
1473         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1474                             GFP_KERNEL);
1475         if (!dp->ports) {
1476                 err = -ENOMEM;
1477                 goto err_destroy_percpu;
1478         }
1479
1480         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1481                 INIT_HLIST_HEAD(&dp->ports[i]);
1482
1483         /* Set up our datapath device. */
1484         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1485         parms.type = OVS_VPORT_TYPE_INTERNAL;
1486         parms.options = NULL;
1487         parms.dp = dp;
1488         parms.port_no = OVSP_LOCAL;
1489         parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1490
1491         ovs_dp_change(dp, a);
1492
1493         /* So far only local changes have been made, now need the lock. */
1494         ovs_lock();
1495
1496         vport = new_vport(&parms);
1497         if (IS_ERR(vport)) {
1498                 err = PTR_ERR(vport);
1499                 if (err == -EBUSY)
1500                         err = -EEXIST;
1501
1502                 if (err == -EEXIST) {
1503                         /* An outdated user space instance that does not understand
1504                          * the concept of user_features has attempted to create a new
1505                          * datapath and is likely to reuse it. Drop all user features.
1506                          */
1507                         if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1508                                 ovs_dp_reset_user_features(skb, info);
1509                 }
1510
1511                 goto err_destroy_ports_array;
1512         }
1513
1514         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1515                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1516         BUG_ON(err < 0);
1517
1518         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1519         list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1520
1521         ovs_unlock();
1522
1523         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1524         return 0;
1525
1526 err_destroy_ports_array:
1527         ovs_unlock();
1528         kfree(dp->ports);
1529 err_destroy_percpu:
1530         free_percpu(dp->stats_percpu);
1531 err_destroy_table:
1532         ovs_flow_tbl_destroy(&dp->table);
1533 err_free_dp:
1534         release_net(ovs_dp_get_net(dp));
1535         kfree(dp);
1536 err_free_reply:
1537         kfree_skb(reply);
1538 err:
1539         return err;
1540 }
1541
1542 /* Called with ovs_mutex. */
1543 static void __dp_destroy(struct datapath *dp)
1544 {
1545         int i;
1546
1547         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1548                 struct vport *vport;
1549                 struct hlist_node *n;
1550
1551                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1552                         if (vport->port_no != OVSP_LOCAL)
1553                                 ovs_dp_detach_port(vport);
1554         }
1555
1556         list_del_rcu(&dp->list_node);
1557
1558         /* OVSP_LOCAL is datapath internal port. We need to make sure that
1559          * all ports in datapath are destroyed first before freeing datapath.
1560          */
1561         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1562
1563         /* RCU destroy the flow table */
1564         call_rcu(&dp->rcu, destroy_dp_rcu);
1565 }
1566
1567 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1568 {
1569         struct sk_buff *reply;
1570         struct datapath *dp;
1571         int err;
1572
1573         reply = ovs_dp_cmd_alloc_info(info);
1574         if (!reply)
1575                 return -ENOMEM;
1576
1577         ovs_lock();
1578         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1579         err = PTR_ERR(dp);
1580         if (IS_ERR(dp))
1581                 goto err_unlock_free;
1582
1583         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1584                                    info->snd_seq, 0, OVS_DP_CMD_DEL);
1585         BUG_ON(err < 0);
1586
1587         __dp_destroy(dp);
1588
1589         ovs_unlock();
1590         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1591         return 0;
1592
1593 err_unlock_free:
1594         ovs_unlock();
1595         kfree_skb(reply);
1596         return err;
1597 }
1598
1599 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1600 {
1601         struct sk_buff *reply;
1602         struct datapath *dp;
1603         int err;
1604
1605         reply = ovs_dp_cmd_alloc_info(info);
1606         if (!reply)
1607                 return -ENOMEM;
1608
1609         ovs_lock();
1610         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1611         err = PTR_ERR(dp);
1612         if (IS_ERR(dp))
1613                 goto err_unlock_free;
1614
1615         ovs_dp_change(dp, info->attrs);
1616
1617         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1618                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1619         BUG_ON(err < 0);
1620
1621         ovs_unlock();
1622         ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
1623         return 0;
1624
1625 err_unlock_free:
1626         ovs_unlock();
1627         kfree_skb(reply);
1628         return err;
1629 }
1630
1631 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1632 {
1633         struct sk_buff *reply;
1634         struct datapath *dp;
1635         int err;
1636
1637         reply = ovs_dp_cmd_alloc_info(info);
1638         if (!reply)
1639                 return -ENOMEM;
1640
1641         rcu_read_lock();
1642         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1643         if (IS_ERR(dp)) {
1644                 err = PTR_ERR(dp);
1645                 goto err_unlock_free;
1646         }
1647         err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1648                                    info->snd_seq, 0, OVS_DP_CMD_NEW);
1649         BUG_ON(err < 0);
1650         rcu_read_unlock();
1651
1652         return genlmsg_reply(reply, info);
1653
1654 err_unlock_free:
1655         rcu_read_unlock();
1656         kfree_skb(reply);
1657         return err;
1658 }
1659
1660 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1661 {
1662         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1663         struct datapath *dp;
1664         int skip = cb->args[0];
1665         int i = 0;
1666
1667         rcu_read_lock();
1668         list_for_each_entry_rcu(dp, &ovs_net->dps, list_node) {
1669                 if (i >= skip &&
1670                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1671                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1672                                          OVS_DP_CMD_NEW) < 0)
1673                         break;
1674                 i++;
1675         }
1676         rcu_read_unlock();
1677
1678         cb->args[0] = i;
1679
1680         return skb->len;
1681 }
1682
1683 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1684         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1685         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1686         [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1687 };
1688
1689 static struct genl_ops dp_datapath_genl_ops[] = {
1690         { .cmd = OVS_DP_CMD_NEW,
1691           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1692           .policy = datapath_policy,
1693           .doit = ovs_dp_cmd_new
1694         },
1695         { .cmd = OVS_DP_CMD_DEL,
1696           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1697           .policy = datapath_policy,
1698           .doit = ovs_dp_cmd_del
1699         },
1700         { .cmd = OVS_DP_CMD_GET,
1701           .flags = 0,               /* OK for unprivileged users. */
1702           .policy = datapath_policy,
1703           .doit = ovs_dp_cmd_get,
1704           .dumpit = ovs_dp_cmd_dump
1705         },
1706         { .cmd = OVS_DP_CMD_SET,
1707           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1708           .policy = datapath_policy,
1709           .doit = ovs_dp_cmd_set,
1710         },
1711 };
1712
1713 static struct genl_family dp_datapath_genl_family = {
1714         .id = GENL_ID_GENERATE,
1715         .hdrsize = sizeof(struct ovs_header),
1716         .name = OVS_DATAPATH_FAMILY,
1717         .version = OVS_DATAPATH_VERSION,
1718         .maxattr = OVS_DP_ATTR_MAX,
1719         .netnsok = true,
1720         .parallel_ops = true,
1721         .ops = dp_datapath_genl_ops,
1722         .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1723         .mcgrps = &ovs_dp_datapath_multicast_group,
1724         .n_mcgrps = 1,
1725 };
1726
1727 /* Called with ovs_mutex or RCU read lock. */
1728 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1729                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1730 {
1731         struct ovs_header *ovs_header;
1732         struct ovs_vport_stats vport_stats;
1733         int err;
1734
1735         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1736                                  flags, cmd);
1737         if (!ovs_header)
1738                 return -EMSGSIZE;
1739
1740         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1741
1742         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1743             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1744             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)))
1745                 goto nla_put_failure;
1746
1747         ovs_vport_get_stats(vport, &vport_stats);
1748         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1749                     &vport_stats))
1750                 goto nla_put_failure;
1751
1752         if (ovs_vport_get_upcall_portids(vport, skb))
1753                 goto nla_put_failure;
1754
1755         err = ovs_vport_get_options(vport, skb);
1756         if (err == -EMSGSIZE)
1757                 goto error;
1758
1759         return genlmsg_end(skb, ovs_header);
1760
1761 nla_put_failure:
1762         err = -EMSGSIZE;
1763 error:
1764         genlmsg_cancel(skb, ovs_header);
1765         return err;
1766 }
1767
1768 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1769 {
1770         return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1771 }
1772
1773 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1774 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1775                                          u32 seq, u8 cmd)
1776 {
1777         struct sk_buff *skb;
1778         int retval;
1779
1780         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1781         if (!skb)
1782                 return ERR_PTR(-ENOMEM);
1783
1784         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1785         BUG_ON(retval < 0);
1786
1787         return skb;
1788 }
1789
1790 /* Called with ovs_mutex or RCU read lock. */
1791 static struct vport *lookup_vport(struct net *net,
1792                                   struct ovs_header *ovs_header,
1793                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1794 {
1795         struct datapath *dp;
1796         struct vport *vport;
1797
1798         if (a[OVS_VPORT_ATTR_NAME]) {
1799                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1800                 if (!vport)
1801                         return ERR_PTR(-ENODEV);
1802                 if (ovs_header->dp_ifindex &&
1803                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1804                         return ERR_PTR(-ENODEV);
1805                 return vport;
1806         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1807                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1808
1809                 if (port_no >= DP_MAX_PORTS)
1810                         return ERR_PTR(-EFBIG);
1811
1812                 dp = get_dp(net, ovs_header->dp_ifindex);
1813                 if (!dp)
1814                         return ERR_PTR(-ENODEV);
1815
1816                 vport = ovs_vport_ovsl_rcu(dp, port_no);
1817                 if (!vport)
1818                         return ERR_PTR(-ENODEV);
1819                 return vport;
1820         } else
1821                 return ERR_PTR(-EINVAL);
1822 }
1823
1824 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1825 {
1826         struct nlattr **a = info->attrs;
1827         struct ovs_header *ovs_header = info->userhdr;
1828         struct vport_parms parms;
1829         struct sk_buff *reply;
1830         struct vport *vport;
1831         struct datapath *dp;
1832         u32 port_no;
1833         int err;
1834
1835         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1836             !a[OVS_VPORT_ATTR_UPCALL_PID])
1837                 return -EINVAL;
1838
1839         port_no = a[OVS_VPORT_ATTR_PORT_NO]
1840                 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
1841         if (port_no >= DP_MAX_PORTS)
1842                 return -EFBIG;
1843
1844         reply = ovs_vport_cmd_alloc_info();
1845         if (!reply)
1846                 return -ENOMEM;
1847
1848         ovs_lock();
1849         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1850         err = -ENODEV;
1851         if (!dp)
1852                 goto exit_unlock_free;
1853
1854         if (port_no) {
1855                 vport = ovs_vport_ovsl(dp, port_no);
1856                 err = -EBUSY;
1857                 if (vport)
1858                         goto exit_unlock_free;
1859         } else {
1860                 for (port_no = 1; ; port_no++) {
1861                         if (port_no >= DP_MAX_PORTS) {
1862                                 err = -EFBIG;
1863                                 goto exit_unlock_free;
1864                         }
1865                         vport = ovs_vport_ovsl(dp, port_no);
1866                         if (!vport)
1867                                 break;
1868                 }
1869         }
1870
1871         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1872         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1873         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1874         parms.dp = dp;
1875         parms.port_no = port_no;
1876         parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
1877
1878         vport = new_vport(&parms);
1879         err = PTR_ERR(vport);
1880         if (IS_ERR(vport))
1881                 goto exit_unlock_free;
1882
1883         err = 0;
1884         if (a[OVS_VPORT_ATTR_STATS])
1885                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
1886
1887         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1888                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1889         BUG_ON(err < 0);
1890         ovs_unlock();
1891
1892         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1893         return 0;
1894
1895 exit_unlock_free:
1896         ovs_unlock();
1897         kfree_skb(reply);
1898         return err;
1899 }
1900
1901 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
1902 {
1903         struct nlattr **a = info->attrs;
1904         struct sk_buff *reply;
1905         struct vport *vport;
1906         int err;
1907
1908         reply = ovs_vport_cmd_alloc_info();
1909         if (!reply)
1910                 return -ENOMEM;
1911
1912         ovs_lock();
1913         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1914         err = PTR_ERR(vport);
1915         if (IS_ERR(vport))
1916                 goto exit_unlock_free;
1917
1918         if (a[OVS_VPORT_ATTR_TYPE] &&
1919             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1920                 err = -EINVAL;
1921                 goto exit_unlock_free;
1922         }
1923
1924         if (a[OVS_VPORT_ATTR_OPTIONS]) {
1925                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
1926                 if (err)
1927                         goto exit_unlock_free;
1928         }
1929
1930         if (a[OVS_VPORT_ATTR_STATS])
1931                 ovs_vport_set_stats(vport, nla_data(a[OVS_VPORT_ATTR_STATS]));
1932
1933
1934         if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
1935                 err = ovs_vport_set_upcall_portids(vport,
1936                                                    a[OVS_VPORT_ATTR_UPCALL_PID]);
1937                 if (err)
1938                         goto exit_unlock_free;
1939         }
1940
1941         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1942                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1943         BUG_ON(err < 0);
1944         ovs_unlock();
1945
1946         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1947         return 0;
1948
1949 exit_unlock_free:
1950         ovs_unlock();
1951         kfree_skb(reply);
1952         return err;
1953 }
1954
1955 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
1956 {
1957         struct nlattr **a = info->attrs;
1958         struct sk_buff *reply;
1959         struct vport *vport;
1960         int err;
1961
1962         reply = ovs_vport_cmd_alloc_info();
1963         if (!reply)
1964                 return -ENOMEM;
1965
1966         ovs_lock();
1967         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1968         err = PTR_ERR(vport);
1969         if (IS_ERR(vport))
1970                 goto exit_unlock_free;
1971
1972         if (vport->port_no == OVSP_LOCAL) {
1973                 err = -EINVAL;
1974                 goto exit_unlock_free;
1975         }
1976
1977         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1978                                       info->snd_seq, 0, OVS_VPORT_CMD_DEL);
1979         BUG_ON(err < 0);
1980         ovs_dp_detach_port(vport);
1981         ovs_unlock();
1982
1983         ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
1984         return 0;
1985
1986 exit_unlock_free:
1987         ovs_unlock();
1988         kfree_skb(reply);
1989         return err;
1990 }
1991
1992 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
1993 {
1994         struct nlattr **a = info->attrs;
1995         struct ovs_header *ovs_header = info->userhdr;
1996         struct sk_buff *reply;
1997         struct vport *vport;
1998         int err;
1999
2000         reply = ovs_vport_cmd_alloc_info();
2001         if (!reply)
2002                 return -ENOMEM;
2003
2004         rcu_read_lock();
2005         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2006         err = PTR_ERR(vport);
2007         if (IS_ERR(vport))
2008                 goto exit_unlock_free;
2009         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
2010                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
2011         BUG_ON(err < 0);
2012         rcu_read_unlock();
2013
2014         return genlmsg_reply(reply, info);
2015
2016 exit_unlock_free:
2017         rcu_read_unlock();
2018         kfree_skb(reply);
2019         return err;
2020 }
2021
2022 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2023 {
2024         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2025         struct datapath *dp;
2026         int bucket = cb->args[0], skip = cb->args[1];
2027         int i, j = 0;
2028
2029         rcu_read_lock();
2030         dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2031         if (!dp) {
2032                 rcu_read_unlock();
2033                 return -ENODEV;
2034         }
2035         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2036                 struct vport *vport;
2037
2038                 j = 0;
2039                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2040                         if (j >= skip &&
2041                             ovs_vport_cmd_fill_info(vport, skb,
2042                                                     NETLINK_CB(cb->skb).portid,
2043                                                     cb->nlh->nlmsg_seq,
2044                                                     NLM_F_MULTI,
2045                                                     OVS_VPORT_CMD_NEW) < 0)
2046                                 goto out;
2047
2048                         j++;
2049                 }
2050                 skip = 0;
2051         }
2052 out:
2053         rcu_read_unlock();
2054
2055         cb->args[0] = i;
2056         cb->args[1] = j;
2057
2058         return skb->len;
2059 }
2060
2061 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2062         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2063         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2064         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2065         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2066         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2067         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2068 };
2069
2070 static struct genl_ops dp_vport_genl_ops[] = {
2071         { .cmd = OVS_VPORT_CMD_NEW,
2072           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2073           .policy = vport_policy,
2074           .doit = ovs_vport_cmd_new
2075         },
2076         { .cmd = OVS_VPORT_CMD_DEL,
2077           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2078           .policy = vport_policy,
2079           .doit = ovs_vport_cmd_del
2080         },
2081         { .cmd = OVS_VPORT_CMD_GET,
2082           .flags = 0,               /* OK for unprivileged users. */
2083           .policy = vport_policy,
2084           .doit = ovs_vport_cmd_get,
2085           .dumpit = ovs_vport_cmd_dump
2086         },
2087         { .cmd = OVS_VPORT_CMD_SET,
2088           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2089           .policy = vport_policy,
2090           .doit = ovs_vport_cmd_set,
2091         },
2092 };
2093
2094 struct genl_family dp_vport_genl_family = {
2095         .id = GENL_ID_GENERATE,
2096         .hdrsize = sizeof(struct ovs_header),
2097         .name = OVS_VPORT_FAMILY,
2098         .version = OVS_VPORT_VERSION,
2099         .maxattr = OVS_VPORT_ATTR_MAX,
2100         .netnsok = true,
2101         .parallel_ops = true,
2102         .ops = dp_vport_genl_ops,
2103         .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2104         .mcgrps = &ovs_dp_vport_multicast_group,
2105         .n_mcgrps = 1,
2106 };
2107
2108 static struct genl_family *dp_genl_families[] = {
2109         &dp_datapath_genl_family,
2110         &dp_vport_genl_family,
2111         &dp_flow_genl_family,
2112         &dp_packet_genl_family,
2113 };
2114
2115 static void dp_unregister_genl(int n_families)
2116 {
2117         int i;
2118
2119         for (i = 0; i < n_families; i++)
2120                 genl_unregister_family(dp_genl_families[i]);
2121 }
2122
2123 static int dp_register_genl(void)
2124 {
2125         int err;
2126         int i;
2127
2128         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2129
2130                 err = genl_register_family(dp_genl_families[i]);
2131                 if (err)
2132                         goto error;
2133         }
2134
2135         return 0;
2136
2137 error:
2138         dp_unregister_genl(i);
2139         return err;
2140 }
2141
2142 static int __net_init ovs_init_net(struct net *net)
2143 {
2144         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2145
2146         INIT_LIST_HEAD(&ovs_net->dps);
2147         INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2148         return 0;
2149 }
2150
2151 static void __net_exit ovs_exit_net(struct net *net)
2152 {
2153         struct datapath *dp, *dp_next;
2154         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2155
2156         ovs_lock();
2157         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2158                 __dp_destroy(dp);
2159         ovs_unlock();
2160
2161         cancel_work_sync(&ovs_net->dp_notify_work);
2162 }
2163
2164 static struct pernet_operations ovs_net_ops = {
2165         .init = ovs_init_net,
2166         .exit = ovs_exit_net,
2167         .id   = &ovs_net_id,
2168         .size = sizeof(struct ovs_net),
2169 };
2170
2171 DEFINE_COMPAT_PNET_REG_FUNC(device);
2172
2173 static int __init dp_init(void)
2174 {
2175         int err;
2176
2177         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2178
2179         pr_info("Open vSwitch switching datapath %s, built "__DATE__" "__TIME__"\n",
2180                 VERSION);
2181
2182         err = ovs_flow_init();
2183         if (err)
2184                 goto error;
2185
2186         err = ovs_vport_init();
2187         if (err)
2188                 goto error_flow_exit;
2189
2190         err = register_pernet_device(&ovs_net_ops);
2191         if (err)
2192                 goto error_vport_exit;
2193
2194         err = register_netdevice_notifier(&ovs_dp_device_notifier);
2195         if (err)
2196                 goto error_netns_exit;
2197
2198         err = dp_register_genl();
2199         if (err < 0)
2200                 goto error_unreg_notifier;
2201
2202         return 0;
2203
2204 error_unreg_notifier:
2205         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2206 error_netns_exit:
2207         unregister_pernet_device(&ovs_net_ops);
2208 error_vport_exit:
2209         ovs_vport_exit();
2210 error_flow_exit:
2211         ovs_flow_exit();
2212 error:
2213         return err;
2214 }
2215
2216 static void dp_cleanup(void)
2217 {
2218         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2219         unregister_netdevice_notifier(&ovs_dp_device_notifier);
2220         unregister_pernet_device(&ovs_net_ops);
2221         rcu_barrier();
2222         ovs_vport_exit();
2223         ovs_flow_exit();
2224 }
2225
2226 module_init(dp_init);
2227 module_exit(dp_cleanup);
2228
2229 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2230 MODULE_LICENSE("GPL");
2231 MODULE_VERSION(VERSION);