2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "connectivity.h"
28 #include "dp-packet.h"
30 #include "openvswitch/hmap.h"
34 #include "ofproto/ofproto-dpif.h"
35 #include "ofproto/ofproto-dpif-rid.h"
36 #include "ofproto/ofproto-provider.h"
37 #include "openvswitch/dynamic-string.h"
38 #include "openvswitch/list.h"
39 #include "openvswitch/match.h"
40 #include "openvswitch/ofp-actions.h"
41 #include "openvswitch/ofp-util.h"
42 #include "openvswitch/ofpbuf.h"
43 #include "openvswitch/vlog.h"
45 #include "poll-loop.h"
47 #include "openvswitch/shash.h"
52 VLOG_DEFINE_THIS_MODULE(bond);
54 static struct ovs_rwlock rwlock = OVS_RWLOCK_INITIALIZER;
55 static struct hmap all_bonds__ = HMAP_INITIALIZER(&all_bonds__);
56 static struct hmap *const all_bonds OVS_GUARDED_BY(rwlock) = &all_bonds__;
58 /* Bit-mask for hashing a flow down to a bucket. */
59 #define BOND_MASK 0xff
60 #define BOND_BUCKETS (BOND_MASK + 1)
62 /* A hash bucket for mapping a flow to a slave.
63 * "struct bond" has an array of BOND_BUCKETS of these. */
65 struct bond_slave *slave; /* Assigned slave, NULL if unassigned. */
66 uint64_t tx_bytes /* Count of bytes recently transmitted. */
67 OVS_GUARDED_BY(rwlock);
68 struct ovs_list list_node; /* In bond_slave's 'entries' list. */
72 * 'pr_rule' is the post-recirculation rule for this entry.
73 * 'pr_tx_bytes' is the most recently seen statistics for 'pr_rule', which
74 * is used to determine delta (applied to 'tx_bytes' above.) */
76 uint64_t pr_tx_bytes OVS_GUARDED_BY(rwlock);
79 /* A bond slave, that is, one of the links comprising a bond. */
81 struct hmap_node hmap_node; /* In struct bond's slaves hmap. */
82 struct ovs_list list_node; /* In struct bond's enabled_slaves list. */
83 struct bond *bond; /* The bond that contains this slave. */
84 void *aux; /* Client-provided handle for this slave. */
86 struct netdev *netdev; /* Network device, owned by the client. */
87 uint64_t change_seq; /* Tracks changes in 'netdev'. */
88 ofp_port_t ofp_port; /* OpenFlow port number. */
89 char *name; /* Name (a copy of netdev_get_name(netdev)). */
92 long long delay_expires; /* Time after which 'enabled' may change. */
93 bool enabled; /* May be chosen for flows? */
94 bool may_enable; /* Client considers this slave bondable. */
96 /* Rebalancing info. Used only by bond_rebalance(). */
97 struct ovs_list bal_node; /* In bond_rebalance()'s 'bals' list. */
98 struct ovs_list entries; /* 'struct bond_entry's assigned here. */
99 uint64_t tx_bytes; /* Sum across 'tx_bytes' of entries. */
102 /* A bond, that is, a set of network devices grouped to improve performance or
105 struct hmap_node hmap_node; /* In 'all_bonds' hmap. */
106 char *name; /* Name provided by client. */
107 struct ofproto_dpif *ofproto; /* The bridge this bond belongs to. */
114 * Any reader or writer of 'enabled_slaves' must hold 'mutex'.
115 * (To prevent the bond_slave from disappearing they must also hold
117 struct ovs_mutex mutex OVS_ACQ_AFTER(rwlock);
118 struct ovs_list enabled_slaves OVS_GUARDED; /* Contains struct bond_slaves. */
121 enum bond_mode balance; /* Balancing mode, one of BM_*. */
122 struct bond_slave *active_slave;
123 int updelay, downdelay; /* Delay before slave goes up/down, in ms. */
124 enum lacp_status lacp_status; /* Status of LACP negotiations. */
125 bool bond_revalidate; /* True if flows need revalidation. */
126 uint32_t basis; /* Basis for flow hash function. */
128 /* SLB specific bonding info. */
129 struct bond_entry *hash; /* An array of BOND_BUCKETS elements. */
130 int rebalance_interval; /* Interval between rebalances, in ms. */
131 long long int next_rebalance; /* Next rebalancing time. */
132 bool send_learning_packets;
133 uint32_t recirc_id; /* Non zero if recirculation can be used.*/
134 struct hmap pr_rule_ops; /* Helps to maintain post recirculation rules.*/
136 /* Store active slave to OVSDB. */
137 bool active_slave_changed; /* Set to true whenever the bond changes
138 active slave. It will be reset to false
139 after it is stored into OVSDB */
141 /* Interface name may not be persistent across an OS reboot, use
142 * MAC address for identifing the active slave */
143 struct eth_addr active_slave_mac;
144 /* The MAC address of the active interface. */
145 /* Legacy compatibility. */
146 bool lacp_fallback_ab; /* Fallback to active-backup on LACP failure. */
148 struct ovs_refcount ref_cnt;
151 /* What to do with an bond_recirc_rule. */
153 ADD, /* Add the rule to ofproto's flow table. */
154 DEL, /* Delete the rule from the ofproto's flow table. */
157 /* A rule to add to or delete from ofproto's internal flow table. */
158 struct bond_pr_rule_op {
159 struct hmap_node hmap_node;
161 ofp_port_t out_ofport;
163 struct rule **pr_rule;
166 static void bond_entry_reset(struct bond *) OVS_REQ_WRLOCK(rwlock);
167 static struct bond_slave *bond_slave_lookup(struct bond *, const void *slave_)
168 OVS_REQ_RDLOCK(rwlock);
169 static void bond_enable_slave(struct bond_slave *, bool enable)
170 OVS_REQ_WRLOCK(rwlock);
171 static void bond_link_status_update(struct bond_slave *)
172 OVS_REQ_WRLOCK(rwlock);
173 static void bond_choose_active_slave(struct bond *)
174 OVS_REQ_WRLOCK(rwlock);
175 static unsigned int bond_hash_src(const struct eth_addr mac,
176 uint16_t vlan, uint32_t basis);
177 static unsigned int bond_hash_tcp(const struct flow *, uint16_t vlan,
179 static struct bond_entry *lookup_bond_entry(const struct bond *,
182 OVS_REQ_RDLOCK(rwlock);
183 static struct bond_slave *get_enabled_slave(struct bond *)
184 OVS_REQ_RDLOCK(rwlock);
185 static struct bond_slave *choose_output_slave(const struct bond *,
187 struct flow_wildcards *,
189 OVS_REQ_RDLOCK(rwlock);
191 /* Attempts to parse 's' as the name of a bond balancing mode. If successful,
192 * stores the mode in '*balance' and returns true. Otherwise returns false
193 * without modifying '*balance'. */
195 bond_mode_from_string(enum bond_mode *balance, const char *s)
197 if (!strcmp(s, bond_mode_to_string(BM_TCP))) {
199 } else if (!strcmp(s, bond_mode_to_string(BM_SLB))) {
201 } else if (!strcmp(s, bond_mode_to_string(BM_AB))) {
209 /* Returns a string representing 'balance'. */
211 bond_mode_to_string(enum bond_mode balance) {
214 return "balance-tcp";
216 return "balance-slb";
218 return "active-backup";
224 /* Creates and returns a new bond whose configuration is initially taken from
227 * The caller should register each slave on the new bond by calling
228 * bond_slave_register(). */
230 bond_create(const struct bond_settings *s, struct ofproto_dpif *ofproto)
234 bond = xzalloc(sizeof *bond);
235 bond->ofproto = ofproto;
236 hmap_init(&bond->slaves);
237 ovs_list_init(&bond->enabled_slaves);
238 ovs_mutex_init(&bond->mutex);
239 ovs_refcount_init(&bond->ref_cnt);
240 hmap_init(&bond->pr_rule_ops);
242 bond_reconfigure(bond, s);
247 bond_ref(const struct bond *bond_)
249 struct bond *bond = CONST_CAST(struct bond *, bond_);
252 ovs_refcount_ref(&bond->ref_cnt);
259 bond_unref(struct bond *bond)
261 struct bond_pr_rule_op *pr_op;
262 struct bond_slave *slave;
264 if (!bond || ovs_refcount_unref_relaxed(&bond->ref_cnt) != 1) {
268 ovs_rwlock_wrlock(&rwlock);
269 hmap_remove(all_bonds, &bond->hmap_node);
270 ovs_rwlock_unlock(&rwlock);
272 HMAP_FOR_EACH_POP (slave, hmap_node, &bond->slaves) {
273 /* Client owns 'slave->netdev'. */
277 hmap_destroy(&bond->slaves);
279 ovs_mutex_destroy(&bond->mutex);
283 HMAP_FOR_EACH_POP (pr_op, hmap_node, &bond->pr_rule_ops) {
286 hmap_destroy(&bond->pr_rule_ops);
288 if (bond->recirc_id) {
289 recirc_free_id(bond->recirc_id);
296 add_pr_rule(struct bond *bond, const struct match *match,
297 ofp_port_t out_ofport, struct rule **rule)
299 uint32_t hash = match_hash(match, 0);
300 struct bond_pr_rule_op *pr_op;
302 HMAP_FOR_EACH_WITH_HASH(pr_op, hmap_node, hash, &bond->pr_rule_ops) {
303 if (match_equal(&pr_op->match, match)) {
305 pr_op->out_ofport = out_ofport;
306 pr_op->pr_rule = rule;
311 pr_op = xmalloc(sizeof *pr_op);
312 pr_op->match = *match;
314 pr_op->out_ofport = out_ofport;
315 pr_op->pr_rule = rule;
316 hmap_insert(&bond->pr_rule_ops, &pr_op->hmap_node, hash);
320 update_recirc_rules(struct bond *bond)
321 OVS_REQ_WRLOCK(rwlock)
324 struct bond_pr_rule_op *pr_op, *next_op;
325 uint64_t ofpacts_stub[128 / 8];
326 struct ofpbuf ofpacts;
329 ofpbuf_use_stub(&ofpacts, ofpacts_stub, sizeof ofpacts_stub);
331 HMAP_FOR_EACH(pr_op, hmap_node, &bond->pr_rule_ops) {
335 if (bond->hash && bond->recirc_id) {
336 for (i = 0; i < BOND_BUCKETS; i++) {
337 struct bond_slave *slave = bond->hash[i].slave;
340 match_init_catchall(&match);
341 match_set_recirc_id(&match, bond->recirc_id);
342 match_set_dp_hash_masked(&match, i, BOND_MASK);
344 add_pr_rule(bond, &match, slave->ofp_port,
345 &bond->hash[i].pr_rule);
350 HMAP_FOR_EACH_SAFE(pr_op, next_op, hmap_node, &bond->pr_rule_ops) {
354 ofpbuf_clear(&ofpacts);
355 ofpact_put_OUTPUT(&ofpacts)->port = pr_op->out_ofport;
356 error = ofproto_dpif_add_internal_flow(bond->ofproto,
358 RECIRC_RULE_PRIORITY, 0,
359 &ofpacts, pr_op->pr_rule);
361 char *err_s = match_to_string(&pr_op->match,
362 RECIRC_RULE_PRIORITY);
364 VLOG_ERR("failed to add post recirculation flow %s", err_s);
370 error = ofproto_dpif_delete_internal_flow(bond->ofproto,
372 RECIRC_RULE_PRIORITY);
374 char *err_s = match_to_string(&pr_op->match,
375 RECIRC_RULE_PRIORITY);
377 VLOG_ERR("failed to remove post recirculation flow %s", err_s);
381 hmap_remove(&bond->pr_rule_ops, &pr_op->hmap_node);
382 *pr_op->pr_rule = NULL;
388 ofpbuf_uninit(&ofpacts);
392 /* Updates 'bond''s overall configuration to 's'.
394 * The caller should register each slave on 'bond' by calling
395 * bond_slave_register(). This is optional if none of the slaves'
396 * configuration has changed. In any case it can't hurt.
398 * Returns true if the configuration has changed in such a way that requires
402 bond_reconfigure(struct bond *bond, const struct bond_settings *s)
404 bool revalidate = false;
406 ovs_rwlock_wrlock(&rwlock);
407 if (!bond->name || strcmp(bond->name, s->name)) {
409 hmap_remove(all_bonds, &bond->hmap_node);
412 bond->name = xstrdup(s->name);
413 hmap_insert(all_bonds, &bond->hmap_node, hash_string(bond->name, 0));
416 bond->updelay = s->up_delay;
417 bond->downdelay = s->down_delay;
419 if (bond->lacp_fallback_ab != s->lacp_fallback_ab_cfg) {
420 bond->lacp_fallback_ab = s->lacp_fallback_ab_cfg;
424 if (bond->rebalance_interval != s->rebalance_interval) {
425 bond->rebalance_interval = s->rebalance_interval;
429 if (bond->balance != s->balance) {
430 bond->balance = s->balance;
434 if (bond->basis != s->basis) {
435 bond->basis = s->basis;
439 if (bond->bond_revalidate) {
441 bond->bond_revalidate = false;
444 if (bond->balance != BM_AB) {
445 if (!bond->recirc_id) {
446 bond->recirc_id = recirc_alloc_id(bond->ofproto);
448 } else if (bond->recirc_id) {
449 recirc_free_id(bond->recirc_id);
453 if (bond->balance == BM_AB || !bond->hash || revalidate) {
454 bond_entry_reset(bond);
457 bond->active_slave_mac = s->active_slave_mac;
458 bond->active_slave_changed = false;
460 ovs_rwlock_unlock(&rwlock);
464 static struct bond_slave *
465 bond_find_slave_by_mac(const struct bond *bond, const struct eth_addr mac)
467 struct bond_slave *slave;
469 /* Find the last active slave */
470 HMAP_FOR_EACH(slave, hmap_node, &bond->slaves) {
471 struct eth_addr slave_mac;
473 if (netdev_get_etheraddr(slave->netdev, &slave_mac)) {
477 if (eth_addr_equals(slave_mac, mac)) {
486 bond_active_slave_changed(struct bond *bond)
490 netdev_get_etheraddr(bond->active_slave->netdev, &mac);
491 bond->active_slave_mac = mac;
492 bond->active_slave_changed = true;
493 seq_change(connectivity_seq_get());
497 bond_slave_set_netdev__(struct bond_slave *slave, struct netdev *netdev)
498 OVS_REQ_WRLOCK(rwlock)
500 if (slave->netdev != netdev) {
501 slave->netdev = netdev;
502 slave->change_seq = 0;
506 /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an
507 * arbitrary client-provided pointer that uniquely identifies a slave within a
508 * bond. If 'slave_' already exists within 'bond' then this function
509 * reconfigures the existing slave.
511 * 'netdev' must be the network device that 'slave_' represents. It is owned
512 * by the client, so the client must not close it before either unregistering
513 * 'slave_' or destroying 'bond'.
516 bond_slave_register(struct bond *bond, void *slave_,
517 ofp_port_t ofport, struct netdev *netdev)
519 struct bond_slave *slave;
521 ovs_rwlock_wrlock(&rwlock);
522 slave = bond_slave_lookup(bond, slave_);
524 slave = xzalloc(sizeof *slave);
526 hmap_insert(&bond->slaves, &slave->hmap_node, hash_pointer(slave_, 0));
529 slave->ofp_port = ofport;
530 slave->delay_expires = LLONG_MAX;
531 slave->name = xstrdup(netdev_get_name(netdev));
532 bond->bond_revalidate = true;
534 slave->enabled = false;
535 bond_enable_slave(slave, netdev_get_carrier(netdev));
538 bond_slave_set_netdev__(slave, netdev);
541 slave->name = xstrdup(netdev_get_name(netdev));
542 ovs_rwlock_unlock(&rwlock);
545 /* Updates the network device to be used with 'slave_' to 'netdev'.
547 * This is useful if the caller closes and re-opens the network device
548 * registered with bond_slave_register() but doesn't need to change anything
551 bond_slave_set_netdev(struct bond *bond, void *slave_, struct netdev *netdev)
553 struct bond_slave *slave;
555 ovs_rwlock_wrlock(&rwlock);
556 slave = bond_slave_lookup(bond, slave_);
558 bond_slave_set_netdev__(slave, netdev);
560 ovs_rwlock_unlock(&rwlock);
563 /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave
564 * then this function has no effect.
566 * Unregistering a slave invalidates all flows. */
568 bond_slave_unregister(struct bond *bond, const void *slave_)
570 struct bond_slave *slave;
573 ovs_rwlock_wrlock(&rwlock);
574 slave = bond_slave_lookup(bond, slave_);
579 bond->bond_revalidate = true;
580 bond_enable_slave(slave, false);
582 del_active = bond->active_slave == slave;
584 struct bond_entry *e;
585 for (e = bond->hash; e <= &bond->hash[BOND_MASK]; e++) {
586 if (e->slave == slave) {
594 hmap_remove(&bond->slaves, &slave->hmap_node);
595 /* Client owns 'slave->netdev'. */
599 bond_choose_active_slave(bond);
600 bond->send_learning_packets = true;
603 ovs_rwlock_unlock(&rwlock);
606 /* Should be called on each slave in 'bond' before bond_run() to indicate
607 * whether or not 'slave_' may be enabled. This function is intended to allow
608 * other protocols to have some impact on bonding decisions. For example LACP
609 * or high level link monitoring protocols may decide that a given slave should
610 * not be able to send traffic. */
612 bond_slave_set_may_enable(struct bond *bond, void *slave_, bool may_enable)
614 ovs_rwlock_wrlock(&rwlock);
615 bond_slave_lookup(bond, slave_)->may_enable = may_enable;
616 ovs_rwlock_unlock(&rwlock);
619 /* Performs periodic maintenance on 'bond'.
621 * Returns true if the caller should revalidate its flows.
623 * The caller should check bond_should_send_learning_packets() afterward. */
625 bond_run(struct bond *bond, enum lacp_status lacp_status)
627 struct bond_slave *slave;
630 ovs_rwlock_wrlock(&rwlock);
631 if (bond->lacp_status != lacp_status) {
632 bond->lacp_status = lacp_status;
633 bond->bond_revalidate = true;
636 /* Enable slaves based on link status and LACP feedback. */
637 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
638 bond_link_status_update(slave);
639 slave->change_seq = seq_read(connectivity_seq_get());
641 if (!bond->active_slave || !bond->active_slave->enabled) {
642 bond_choose_active_slave(bond);
645 revalidate = bond->bond_revalidate;
646 bond->bond_revalidate = false;
647 ovs_rwlock_unlock(&rwlock);
652 /* Causes poll_block() to wake up when 'bond' needs something to be done. */
654 bond_wait(struct bond *bond)
656 struct bond_slave *slave;
658 ovs_rwlock_rdlock(&rwlock);
659 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
660 if (slave->delay_expires != LLONG_MAX) {
661 poll_timer_wait_until(slave->delay_expires);
664 seq_wait(connectivity_seq_get(), slave->change_seq);
667 if (bond->bond_revalidate) {
668 poll_immediate_wake();
670 ovs_rwlock_unlock(&rwlock);
672 /* We don't wait for bond->next_rebalance because rebalancing can only run
673 * at a flow account checkpoint. ofproto does checkpointing on its own
674 * schedule and bond_rebalance() gets called afterward, so we'd just be
675 * waking up for no purpose. */
678 /* MAC learning table interaction. */
681 may_send_learning_packets(const struct bond *bond)
683 return ((bond->lacp_status == LACP_DISABLED
684 && (bond->balance == BM_SLB || bond->balance == BM_AB))
685 || (bond->lacp_fallback_ab && bond->lacp_status == LACP_CONFIGURED))
686 && bond->active_slave;
689 /* Returns true if 'bond' needs the client to send out packets to assist with
690 * MAC learning on 'bond'. If this function returns true, then the client
691 * should iterate through its MAC learning table for the bridge on which 'bond'
692 * is located. For each MAC that has been learned on a port other than 'bond',
693 * it should call bond_compose_learning_packet().
695 * This function will only return true if 'bond' is in SLB or active-backup
696 * mode and LACP is not negotiated. Otherwise sending learning packets isn't
699 * Calling this function resets the state that it checks. */
701 bond_should_send_learning_packets(struct bond *bond)
705 ovs_rwlock_wrlock(&rwlock);
706 send = bond->send_learning_packets && may_send_learning_packets(bond);
707 bond->send_learning_packets = false;
708 ovs_rwlock_unlock(&rwlock);
712 /* Sends a gratuitous learning packet on 'bond' from 'eth_src' on 'vlan'.
714 * See bond_should_send_learning_packets() for description of usage. The
715 * caller should send the composed packet on the port associated with
716 * port_aux and takes ownership of the returned ofpbuf. */
718 bond_compose_learning_packet(struct bond *bond, const struct eth_addr eth_src,
719 uint16_t vlan, void **port_aux)
721 struct bond_slave *slave;
722 struct dp_packet *packet;
725 ovs_rwlock_rdlock(&rwlock);
726 ovs_assert(may_send_learning_packets(bond));
727 memset(&flow, 0, sizeof flow);
728 flow.dl_src = eth_src;
729 slave = choose_output_slave(bond, &flow, NULL, vlan);
731 packet = dp_packet_new(0);
732 compose_rarp(packet, eth_src);
734 eth_push_vlan(packet, htons(ETH_TYPE_VLAN), htons(vlan));
737 *port_aux = slave->aux;
738 ovs_rwlock_unlock(&rwlock);
742 /* Checks whether a packet that arrived on 'slave_' within 'bond', with an
743 * Ethernet destination address of 'eth_dst', should be admitted.
745 * The return value is one of the following:
747 * - BV_ACCEPT: Admit the packet.
749 * - BV_DROP: Drop the packet.
751 * - BV_DROP_IF_MOVED: Consult the MAC learning table for the packet's
752 * Ethernet source address and VLAN. If there is none, or if the packet
753 * is on the learned port, then admit the packet. If a different port has
754 * been learned, however, drop the packet (and do not use it for MAC
758 bond_check_admissibility(struct bond *bond, const void *slave_,
759 const struct eth_addr eth_dst)
761 enum bond_verdict verdict = BV_DROP;
762 struct bond_slave *slave;
764 ovs_rwlock_rdlock(&rwlock);
765 slave = bond_slave_lookup(bond, slave_);
770 /* LACP bonds have very loose admissibility restrictions because we can
771 * assume the remote switch is aware of the bond and will "do the right
772 * thing". However, as a precaution we drop packets on disabled slaves
773 * because no correctly implemented partner switch should be sending
776 * If LACP is configured, but LACP negotiations have been unsuccessful, we
777 * drop all incoming traffic except if lacp_fallback_ab is enabled. */
778 switch (bond->lacp_status) {
779 case LACP_NEGOTIATED:
780 verdict = slave->enabled ? BV_ACCEPT : BV_DROP;
782 case LACP_CONFIGURED:
783 if (!bond->lacp_fallback_ab) {
790 /* Drop all multicast packets on inactive slaves. */
791 if (eth_addr_is_multicast(eth_dst)) {
792 if (bond->active_slave != slave) {
797 switch (bond->balance) {
799 /* TCP balanced bonds require successful LACP negotiations. Based on the
800 * above check, LACP is off or lacp_fallback_ab is true on this bond.
801 * If lacp_fallback_ab is true fall through to BM_AB case else, we
802 * drop all incoming traffic. */
803 if (!bond->lacp_fallback_ab) {
808 /* Drop all packets which arrive on backup slaves. This is similar to
809 * how Linux bonding handles active-backup bonds. */
810 if (bond->active_slave != slave) {
811 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
813 VLOG_DBG_RL(&rl, "active-backup bond received packet on backup"
814 " slave (%s) destined for " ETH_ADDR_FMT,
815 slave->name, ETH_ADDR_ARGS(eth_dst));
822 /* Drop all packets for which we have learned a different input port,
823 * because we probably sent the packet on one slave and got it back on
824 * the other. Gratuitous ARP packets are an exception to this rule:
825 * the host has moved to another switch. The exception to the
826 * exception is if we locked the learning table to avoid reflections on
828 verdict = BV_DROP_IF_MOVED;
834 ovs_rwlock_unlock(&rwlock);
839 /* Returns the slave (registered on 'bond' by bond_slave_register()) to which
840 * a packet with the given 'flow' and 'vlan' should be forwarded. Returns
841 * NULL if the packet should be dropped because no slaves are enabled.
843 * 'vlan' is not necessarily the same as 'flow->vlan_tci'. First, 'vlan'
844 * should be a VID only (i.e. excluding the PCP bits). Second,
845 * 'flow->vlan_tci' is the VLAN TCI that appeared on the packet (so it will be
846 * nonzero only for trunk ports), whereas 'vlan' is the logical VLAN that the
847 * packet belongs to (so for an access port it will be the access port's VLAN).
849 * If 'wc' is non-NULL, bitwise-OR's 'wc' with the set of bits that were
850 * significant in the selection. At some point earlier, 'wc' should
851 * have been initialized (e.g., by flow_wildcards_init_catchall()).
854 bond_choose_output_slave(struct bond *bond, const struct flow *flow,
855 struct flow_wildcards *wc, uint16_t vlan)
857 struct bond_slave *slave;
860 ovs_rwlock_rdlock(&rwlock);
861 slave = choose_output_slave(bond, flow, wc, vlan);
862 aux = slave ? slave->aux : NULL;
863 ovs_rwlock_unlock(&rwlock);
870 bond_entry_account(struct bond_entry *entry, uint64_t rule_tx_bytes)
871 OVS_REQ_WRLOCK(rwlock)
876 delta = rule_tx_bytes - entry->pr_tx_bytes;
877 entry->tx_bytes += delta;
878 entry->pr_tx_bytes = rule_tx_bytes;
882 /* Maintain bond stats using post recirculation rule byte counters.*/
884 bond_recirculation_account(struct bond *bond)
885 OVS_REQ_WRLOCK(rwlock)
889 for (i=0; i<=BOND_MASK; i++) {
890 struct bond_entry *entry = &bond->hash[i];
891 struct rule *rule = entry->pr_rule;
894 uint64_t n_packets OVS_UNUSED;
895 long long int used OVS_UNUSED;
898 rule->ofproto->ofproto_class->rule_get_stats(
899 rule, &n_packets, &n_bytes, &used);
900 bond_entry_account(entry, n_bytes);
906 bond_may_recirc(const struct bond *bond, uint32_t *recirc_id,
909 if (bond->balance == BM_TCP && bond->recirc_id) {
911 *recirc_id = bond->recirc_id;
914 *hash_bias = bond->basis;
923 bond_update_post_recirc_rules__(struct bond* bond, const bool force)
924 OVS_REQ_WRLOCK(rwlock)
926 struct bond_entry *e;
927 bool update_rules = force; /* Always update rules if caller forces it. */
929 /* Make sure all bond entries are populated */
930 for (e = bond->hash; e <= &bond->hash[BOND_MASK]; e++) {
931 if (!e->slave || !e->slave->enabled) {
933 e->slave = CONTAINER_OF(hmap_random_node(&bond->slaves),
934 struct bond_slave, hmap_node);
935 if (!e->slave->enabled) {
936 e->slave = bond->active_slave;
942 update_recirc_rules(bond);
947 bond_update_post_recirc_rules(struct bond* bond, const bool force)
949 ovs_rwlock_wrlock(&rwlock);
950 bond_update_post_recirc_rules__(bond, force);
951 ovs_rwlock_unlock(&rwlock);
957 bond_is_balanced(const struct bond *bond) OVS_REQ_RDLOCK(rwlock)
959 return bond->rebalance_interval
960 && (bond->balance == BM_SLB || bond->balance == BM_TCP);
963 /* Notifies 'bond' that 'n_bytes' bytes were sent in 'flow' within 'vlan'. */
965 bond_account(struct bond *bond, const struct flow *flow, uint16_t vlan,
968 ovs_rwlock_wrlock(&rwlock);
969 if (bond_is_balanced(bond)) {
970 lookup_bond_entry(bond, flow, vlan)->tx_bytes += n_bytes;
972 ovs_rwlock_unlock(&rwlock);
975 static struct bond_slave *
976 bond_slave_from_bal_node(struct ovs_list *bal) OVS_REQ_RDLOCK(rwlock)
978 return CONTAINER_OF(bal, struct bond_slave, bal_node);
982 log_bals(struct bond *bond, const struct ovs_list *bals)
983 OVS_REQ_RDLOCK(rwlock)
985 if (VLOG_IS_DBG_ENABLED()) {
986 struct ds ds = DS_EMPTY_INITIALIZER;
987 const struct bond_slave *slave;
989 LIST_FOR_EACH (slave, bal_node, bals) {
991 ds_put_char(&ds, ',');
993 ds_put_format(&ds, " %s %"PRIu64"kB",
994 slave->name, slave->tx_bytes / 1024);
996 if (!slave->enabled) {
997 ds_put_cstr(&ds, " (disabled)");
999 if (!ovs_list_is_empty(&slave->entries)) {
1000 struct bond_entry *e;
1002 ds_put_cstr(&ds, " (");
1003 LIST_FOR_EACH (e, list_node, &slave->entries) {
1004 if (&e->list_node != ovs_list_front(&slave->entries)) {
1005 ds_put_cstr(&ds, " + ");
1007 ds_put_format(&ds, "h%"PRIdPTR": %"PRIu64"kB",
1008 e - bond->hash, e->tx_bytes / 1024);
1010 ds_put_cstr(&ds, ")");
1013 VLOG_DBG("bond %s:%s", bond->name, ds_cstr(&ds));
1018 /* Shifts 'hash' from its current slave to 'to'. */
1020 bond_shift_load(struct bond_entry *hash, struct bond_slave *to)
1021 OVS_REQ_WRLOCK(rwlock)
1023 struct bond_slave *from = hash->slave;
1024 struct bond *bond = from->bond;
1025 uint64_t delta = hash->tx_bytes;
1027 VLOG_INFO("bond %s: shift %"PRIu64"kB of load (with hash %"PRIdPTR") "
1028 "from %s to %s (now carrying %"PRIu64"kB and "
1029 "%"PRIu64"kB load, respectively)",
1030 bond->name, delta / 1024, hash - bond->hash,
1031 from->name, to->name,
1032 (from->tx_bytes - delta) / 1024,
1033 (to->tx_bytes + delta) / 1024);
1035 /* Shift load away from 'from' to 'to'. */
1036 from->tx_bytes -= delta;
1037 to->tx_bytes += delta;
1039 /* Arrange for flows to be revalidated. */
1041 bond->bond_revalidate = true;
1044 /* Picks and returns a bond_entry to migrate from 'from' (the most heavily
1045 * loaded bond slave) to a bond slave that has 'to_tx_bytes' bytes of load,
1046 * given that doing so must decrease the ratio of the load on the two slaves by
1047 * at least 0.1. Returns NULL if there is no appropriate entry.
1049 * The list of entries isn't sorted. I don't know of a reason to prefer to
1050 * shift away small hashes or large hashes. */
1051 static struct bond_entry *
1052 choose_entry_to_migrate(const struct bond_slave *from, uint64_t to_tx_bytes)
1053 OVS_REQ_WRLOCK(rwlock)
1055 struct bond_entry *e;
1057 if (ovs_list_is_short(&from->entries)) {
1058 /* 'from' carries no more than one MAC hash, so shifting load away from
1059 * it would be pointless. */
1063 LIST_FOR_EACH (e, list_node, &from->entries) {
1064 uint64_t delta = e->tx_bytes; /* The amount to rebalance. */
1065 uint64_t ideal_tx_bytes = (from->tx_bytes + to_tx_bytes)/2;
1066 /* Note, the ideal traffic is the mid point
1067 * between 'from' and 'to'. This value does
1068 * not change by rebalancing. */
1069 uint64_t new_low; /* The lower bandwidth between 'to' and 'from'
1070 after rebalancing. */
1072 new_low = MIN(from->tx_bytes - delta, to_tx_bytes + delta);
1074 if ((new_low > to_tx_bytes) &&
1075 (new_low - to_tx_bytes >= (ideal_tx_bytes - to_tx_bytes) / 10)) {
1076 /* Only rebalance if the new 'low' is closer to to the mid point,
1077 * and the improvement exceeds 10% of current traffic
1078 * deviation from the ideal split.
1080 * The improvement on the 'high' side is always the same as the
1081 * 'low' side. Thus consider 'low' side is sufficient. */
1089 /* Inserts 'slave' into 'bals' so that descending order of 'tx_bytes' is
1092 insert_bal(struct ovs_list *bals, struct bond_slave *slave)
1094 struct bond_slave *pos;
1096 LIST_FOR_EACH (pos, bal_node, bals) {
1097 if (slave->tx_bytes > pos->tx_bytes) {
1101 ovs_list_insert(&pos->bal_node, &slave->bal_node);
1104 /* Removes 'slave' from its current list and then inserts it into 'bals' so
1105 * that descending order of 'tx_bytes' is maintained. */
1107 reinsert_bal(struct ovs_list *bals, struct bond_slave *slave)
1109 ovs_list_remove(&slave->bal_node);
1110 insert_bal(bals, slave);
1113 /* If 'bond' needs rebalancing, does so.
1115 * The caller should have called bond_account() for each active flow, or in case
1116 * of recirculation is used, have called bond_recirculation_account(bond),
1117 * to ensure that flow data is consistently accounted at this point.
1120 bond_rebalance(struct bond *bond)
1122 struct bond_slave *slave;
1123 struct bond_entry *e;
1124 struct ovs_list bals;
1125 bool rebalanced = false;
1128 ovs_rwlock_wrlock(&rwlock);
1129 if (!bond_is_balanced(bond) || time_msec() < bond->next_rebalance) {
1132 bond->next_rebalance = time_msec() + bond->rebalance_interval;
1134 use_recirc = ofproto_dpif_get_support(bond->ofproto)->odp.recirc &&
1135 bond_may_recirc(bond, NULL, NULL);
1138 bond_recirculation_account(bond);
1141 /* Add each bond_entry to its slave's 'entries' list.
1142 * Compute each slave's tx_bytes as the sum of its entries' tx_bytes. */
1143 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1144 slave->tx_bytes = 0;
1145 ovs_list_init(&slave->entries);
1147 for (e = &bond->hash[0]; e <= &bond->hash[BOND_MASK]; e++) {
1148 if (e->slave && e->tx_bytes) {
1149 e->slave->tx_bytes += e->tx_bytes;
1150 ovs_list_push_back(&e->slave->entries, &e->list_node);
1154 /* Add enabled slaves to 'bals' in descending order of tx_bytes.
1156 * XXX This is O(n**2) in the number of slaves but it could be O(n lg n)
1157 * with a proper list sort algorithm. */
1158 ovs_list_init(&bals);
1159 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1160 if (slave->enabled) {
1161 insert_bal(&bals, slave);
1164 log_bals(bond, &bals);
1166 /* Shift load from the most-loaded slaves to the least-loaded slaves. */
1167 while (!ovs_list_is_short(&bals)) {
1168 struct bond_slave *from = bond_slave_from_bal_node(ovs_list_front(&bals));
1169 struct bond_slave *to = bond_slave_from_bal_node(ovs_list_back(&bals));
1172 overload = from->tx_bytes - to->tx_bytes;
1173 if (overload < to->tx_bytes >> 5 || overload < 100000) {
1174 /* The extra load on 'from' (and all less-loaded slaves), compared
1175 * to that of 'to' (the least-loaded slave), is less than ~3%, or
1176 * it is less than ~1Mbps. No point in rebalancing. */
1180 /* 'from' is carrying significantly more load than 'to'. Pick a hash
1181 * to move from 'from' to 'to'. */
1182 e = choose_entry_to_migrate(from, to->tx_bytes);
1184 bond_shift_load(e, to);
1186 /* Delete element from from->entries.
1188 * We don't add the element to to->hashes. That would only allow
1189 * 'e' to be migrated to another slave in this rebalancing run, and
1190 * there is no point in doing that. */
1191 ovs_list_remove(&e->list_node);
1193 /* Re-sort 'bals'. */
1194 reinsert_bal(&bals, from);
1195 reinsert_bal(&bals, to);
1198 /* Can't usefully migrate anything away from 'from'.
1199 * Don't reconsider it. */
1200 ovs_list_remove(&from->bal_node);
1204 /* Implement exponentially weighted moving average. A weight of 1/2 causes
1205 * historical data to decay to <1% in 7 rebalancing runs. 1,000,000 bytes
1206 * take 20 rebalancing runs to decay to 0 and get deleted entirely. */
1207 for (e = &bond->hash[0]; e <= &bond->hash[BOND_MASK]; e++) {
1211 if (use_recirc && rebalanced) {
1212 bond_update_post_recirc_rules__(bond,true);
1216 ovs_rwlock_unlock(&rwlock);
1219 /* Bonding unixctl user interface functions. */
1221 static struct bond *
1222 bond_find(const char *name) OVS_REQ_RDLOCK(rwlock)
1226 HMAP_FOR_EACH_WITH_HASH (bond, hmap_node, hash_string(name, 0),
1228 if (!strcmp(bond->name, name)) {
1235 static struct bond_slave *
1236 bond_lookup_slave(struct bond *bond, const char *slave_name)
1238 struct bond_slave *slave;
1240 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1241 if (!strcmp(slave->name, slave_name)) {
1249 bond_unixctl_list(struct unixctl_conn *conn,
1250 int argc OVS_UNUSED, const char *argv[] OVS_UNUSED,
1251 void *aux OVS_UNUSED)
1253 struct ds ds = DS_EMPTY_INITIALIZER;
1254 const struct bond *bond;
1256 ds_put_cstr(&ds, "bond\ttype\trecircID\tslaves\n");
1258 ovs_rwlock_rdlock(&rwlock);
1259 HMAP_FOR_EACH (bond, hmap_node, all_bonds) {
1260 const struct bond_slave *slave;
1263 ds_put_format(&ds, "%s\t%s\t%d\t", bond->name,
1264 bond_mode_to_string(bond->balance), bond->recirc_id);
1267 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1269 ds_put_cstr(&ds, ", ");
1271 ds_put_cstr(&ds, slave->name);
1273 ds_put_char(&ds, '\n');
1275 ovs_rwlock_unlock(&rwlock);
1276 unixctl_command_reply(conn, ds_cstr(&ds));
1281 bond_print_details(struct ds *ds, const struct bond *bond)
1282 OVS_REQ_RDLOCK(rwlock)
1284 struct shash slave_shash = SHASH_INITIALIZER(&slave_shash);
1285 const struct shash_node **sorted_slaves = NULL;
1286 const struct bond_slave *slave;
1291 ds_put_format(ds, "---- %s ----\n", bond->name);
1292 ds_put_format(ds, "bond_mode: %s\n",
1293 bond_mode_to_string(bond->balance));
1295 may_recirc = bond_may_recirc(bond, &recirc_id, NULL);
1296 ds_put_format(ds, "bond may use recirculation: %s, Recirc-ID : %d\n",
1297 may_recirc ? "yes" : "no", may_recirc ? recirc_id: -1);
1299 ds_put_format(ds, "bond-hash-basis: %"PRIu32"\n", bond->basis);
1301 ds_put_format(ds, "updelay: %d ms\n", bond->updelay);
1302 ds_put_format(ds, "downdelay: %d ms\n", bond->downdelay);
1304 if (bond_is_balanced(bond)) {
1305 ds_put_format(ds, "next rebalance: %lld ms\n",
1306 bond->next_rebalance - time_msec());
1309 ds_put_cstr(ds, "lacp_status: ");
1310 switch (bond->lacp_status) {
1311 case LACP_NEGOTIATED:
1312 ds_put_cstr(ds, "negotiated\n");
1314 case LACP_CONFIGURED:
1315 ds_put_cstr(ds, "configured\n");
1318 ds_put_cstr(ds, "off\n");
1321 ds_put_cstr(ds, "<unknown>\n");
1325 ds_put_cstr(ds, "active slave mac: ");
1326 ds_put_format(ds, ETH_ADDR_FMT, ETH_ADDR_ARGS(bond->active_slave_mac));
1327 slave = bond_find_slave_by_mac(bond, bond->active_slave_mac);
1328 ds_put_format(ds,"(%s)\n", slave ? slave->name : "none");
1330 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1331 shash_add(&slave_shash, slave->name, slave);
1333 sorted_slaves = shash_sort(&slave_shash);
1335 for (i = 0; i < shash_count(&slave_shash); i++) {
1336 struct bond_entry *be;
1338 slave = sorted_slaves[i]->data;
1341 ds_put_format(ds, "\nslave %s: %s\n",
1342 slave->name, slave->enabled ? "enabled" : "disabled");
1343 if (slave == bond->active_slave) {
1344 ds_put_cstr(ds, "\tactive slave\n");
1346 if (slave->delay_expires != LLONG_MAX) {
1347 ds_put_format(ds, "\t%s expires in %lld ms\n",
1348 slave->enabled ? "downdelay" : "updelay",
1349 slave->delay_expires - time_msec());
1352 ds_put_format(ds, "\tmay_enable: %s\n",
1353 slave->may_enable ? "true" : "false");
1355 if (!bond_is_balanced(bond)) {
1360 for (be = bond->hash; be <= &bond->hash[BOND_MASK]; be++) {
1361 int hash = be - bond->hash;
1364 if (be->slave != slave) {
1368 be_tx_k = be->tx_bytes / 1024;
1370 ds_put_format(ds, "\thash %d: %"PRIu64" kB load\n",
1374 /* XXX How can we list the MACs assigned to hashes of SLB bonds? */
1377 shash_destroy(&slave_shash);
1378 free(sorted_slaves);
1379 ds_put_cstr(ds, "\n");
1383 bond_unixctl_show(struct unixctl_conn *conn,
1384 int argc, const char *argv[],
1385 void *aux OVS_UNUSED)
1387 struct ds ds = DS_EMPTY_INITIALIZER;
1389 ovs_rwlock_rdlock(&rwlock);
1391 const struct bond *bond = bond_find(argv[1]);
1394 unixctl_command_reply_error(conn, "no such bond");
1397 bond_print_details(&ds, bond);
1399 const struct bond *bond;
1401 HMAP_FOR_EACH (bond, hmap_node, all_bonds) {
1402 bond_print_details(&ds, bond);
1406 unixctl_command_reply(conn, ds_cstr(&ds));
1410 ovs_rwlock_unlock(&rwlock);
1414 bond_unixctl_migrate(struct unixctl_conn *conn,
1415 int argc OVS_UNUSED, const char *argv[],
1416 void *aux OVS_UNUSED)
1418 const char *bond_s = argv[1];
1419 const char *hash_s = argv[2];
1420 const char *slave_s = argv[3];
1422 struct bond_slave *slave;
1423 struct bond_entry *entry;
1426 ovs_rwlock_wrlock(&rwlock);
1427 bond = bond_find(bond_s);
1429 unixctl_command_reply_error(conn, "no such bond");
1433 if (bond->balance != BM_SLB) {
1434 unixctl_command_reply_error(conn, "not an SLB bond");
1438 if (strspn(hash_s, "0123456789") == strlen(hash_s)) {
1439 hash = atoi(hash_s) & BOND_MASK;
1441 unixctl_command_reply_error(conn, "bad hash");
1445 slave = bond_lookup_slave(bond, slave_s);
1447 unixctl_command_reply_error(conn, "no such slave");
1451 if (!slave->enabled) {
1452 unixctl_command_reply_error(conn, "cannot migrate to disabled slave");
1456 entry = &bond->hash[hash];
1457 bond->bond_revalidate = true;
1458 entry->slave = slave;
1459 unixctl_command_reply(conn, "migrated");
1462 ovs_rwlock_unlock(&rwlock);
1466 bond_unixctl_set_active_slave(struct unixctl_conn *conn,
1467 int argc OVS_UNUSED, const char *argv[],
1468 void *aux OVS_UNUSED)
1470 const char *bond_s = argv[1];
1471 const char *slave_s = argv[2];
1473 struct bond_slave *slave;
1475 ovs_rwlock_wrlock(&rwlock);
1476 bond = bond_find(bond_s);
1478 unixctl_command_reply_error(conn, "no such bond");
1482 slave = bond_lookup_slave(bond, slave_s);
1484 unixctl_command_reply_error(conn, "no such slave");
1488 if (!slave->enabled) {
1489 unixctl_command_reply_error(conn, "cannot make disabled slave active");
1493 if (bond->active_slave != slave) {
1494 bond->bond_revalidate = true;
1495 bond->active_slave = slave;
1496 VLOG_INFO("bond %s: active interface is now %s",
1497 bond->name, slave->name);
1498 bond->send_learning_packets = true;
1499 unixctl_command_reply(conn, "done");
1500 bond_active_slave_changed(bond);
1502 unixctl_command_reply(conn, "no change");
1505 ovs_rwlock_unlock(&rwlock);
1509 enable_slave(struct unixctl_conn *conn, const char *argv[], bool enable)
1511 const char *bond_s = argv[1];
1512 const char *slave_s = argv[2];
1514 struct bond_slave *slave;
1516 ovs_rwlock_wrlock(&rwlock);
1517 bond = bond_find(bond_s);
1519 unixctl_command_reply_error(conn, "no such bond");
1523 slave = bond_lookup_slave(bond, slave_s);
1525 unixctl_command_reply_error(conn, "no such slave");
1529 bond_enable_slave(slave, enable);
1530 unixctl_command_reply(conn, enable ? "enabled" : "disabled");
1533 ovs_rwlock_unlock(&rwlock);
1537 bond_unixctl_enable_slave(struct unixctl_conn *conn,
1538 int argc OVS_UNUSED, const char *argv[],
1539 void *aux OVS_UNUSED)
1541 enable_slave(conn, argv, true);
1545 bond_unixctl_disable_slave(struct unixctl_conn *conn,
1546 int argc OVS_UNUSED, const char *argv[],
1547 void *aux OVS_UNUSED)
1549 enable_slave(conn, argv, false);
1553 bond_unixctl_hash(struct unixctl_conn *conn, int argc, const char *argv[],
1554 void *aux OVS_UNUSED)
1556 const char *mac_s = argv[1];
1557 const char *vlan_s = argc > 2 ? argv[2] : NULL;
1558 const char *basis_s = argc > 3 ? argv[3] : NULL;
1559 struct eth_addr mac;
1566 if (!ovs_scan(vlan_s, "%u", &vlan)) {
1567 unixctl_command_reply_error(conn, "invalid vlan");
1575 if (!ovs_scan(basis_s, "%"SCNu32, &basis)) {
1576 unixctl_command_reply_error(conn, "invalid basis");
1583 if (ovs_scan(mac_s, ETH_ADDR_SCAN_FMT, ETH_ADDR_SCAN_ARGS(mac))) {
1584 hash = bond_hash_src(mac, vlan, basis) & BOND_MASK;
1586 hash_cstr = xasprintf("%u", hash);
1587 unixctl_command_reply(conn, hash_cstr);
1590 unixctl_command_reply_error(conn, "invalid mac");
1597 unixctl_command_register("bond/list", "", 0, 0, bond_unixctl_list, NULL);
1598 unixctl_command_register("bond/show", "[port]", 0, 1, bond_unixctl_show,
1600 unixctl_command_register("bond/migrate", "port hash slave", 3, 3,
1601 bond_unixctl_migrate, NULL);
1602 unixctl_command_register("bond/set-active-slave", "port slave", 2, 2,
1603 bond_unixctl_set_active_slave, NULL);
1604 unixctl_command_register("bond/enable-slave", "port slave", 2, 2,
1605 bond_unixctl_enable_slave, NULL);
1606 unixctl_command_register("bond/disable-slave", "port slave", 2, 2,
1607 bond_unixctl_disable_slave, NULL);
1608 unixctl_command_register("bond/hash", "mac [vlan] [basis]", 1, 3,
1609 bond_unixctl_hash, NULL);
1613 bond_entry_reset(struct bond *bond)
1615 if (bond->balance != BM_AB) {
1616 size_t hash_len = BOND_BUCKETS * sizeof *bond->hash;
1619 bond->hash = xmalloc(hash_len);
1621 memset(bond->hash, 0, hash_len);
1623 bond->next_rebalance = time_msec() + bond->rebalance_interval;
1630 static struct bond_slave *
1631 bond_slave_lookup(struct bond *bond, const void *slave_)
1633 struct bond_slave *slave;
1635 HMAP_FOR_EACH_IN_BUCKET (slave, hmap_node, hash_pointer(slave_, 0),
1637 if (slave->aux == slave_) {
1646 bond_enable_slave(struct bond_slave *slave, bool enable)
1648 slave->delay_expires = LLONG_MAX;
1649 if (enable != slave->enabled) {
1650 slave->bond->bond_revalidate = true;
1651 slave->enabled = enable;
1653 ovs_mutex_lock(&slave->bond->mutex);
1655 ovs_list_insert(&slave->bond->enabled_slaves, &slave->list_node);
1657 ovs_list_remove(&slave->list_node);
1659 ovs_mutex_unlock(&slave->bond->mutex);
1661 VLOG_INFO("interface %s: %s", slave->name,
1662 slave->enabled ? "enabled" : "disabled");
1667 bond_link_status_update(struct bond_slave *slave)
1669 struct bond *bond = slave->bond;
1672 up = netdev_get_carrier(slave->netdev) && slave->may_enable;
1673 if ((up == slave->enabled) != (slave->delay_expires == LLONG_MAX)) {
1674 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
1675 VLOG_INFO_RL(&rl, "interface %s: link state %s",
1676 slave->name, up ? "up" : "down");
1677 if (up == slave->enabled) {
1678 slave->delay_expires = LLONG_MAX;
1679 VLOG_INFO_RL(&rl, "interface %s: will not be %s",
1680 slave->name, up ? "disabled" : "enabled");
1682 int delay = (bond->lacp_status != LACP_DISABLED ? 0
1683 : up ? bond->updelay : bond->downdelay);
1684 slave->delay_expires = time_msec() + delay;
1686 VLOG_INFO_RL(&rl, "interface %s: will be %s if it stays %s "
1689 up ? "enabled" : "disabled",
1696 if (time_msec() >= slave->delay_expires) {
1697 bond_enable_slave(slave, up);
1702 bond_hash_src(const struct eth_addr mac, uint16_t vlan, uint32_t basis)
1704 return hash_mac(mac, vlan, basis);
1708 bond_hash_tcp(const struct flow *flow, uint16_t vlan, uint32_t basis)
1710 struct flow hash_flow = *flow;
1711 hash_flow.vlan_tci = htons(vlan);
1713 /* The symmetric quality of this hash function is not required, but
1714 * flow_hash_symmetric_l4 already exists, and is sufficient for our
1715 * purposes, so we use it out of convenience. */
1716 return flow_hash_symmetric_l4(&hash_flow, basis);
1720 bond_hash(const struct bond *bond, const struct flow *flow, uint16_t vlan)
1722 ovs_assert(bond->balance == BM_TCP || bond->balance == BM_SLB);
1724 return (bond->balance == BM_TCP
1725 ? bond_hash_tcp(flow, vlan, bond->basis)
1726 : bond_hash_src(flow->dl_src, vlan, bond->basis));
1729 static struct bond_entry *
1730 lookup_bond_entry(const struct bond *bond, const struct flow *flow,
1733 return &bond->hash[bond_hash(bond, flow, vlan) & BOND_MASK];
1736 /* Selects and returns an enabled slave from the 'enabled_slaves' list
1737 * in a round-robin fashion. If the 'enabled_slaves' list is empty,
1739 static struct bond_slave *
1740 get_enabled_slave(struct bond *bond)
1742 struct ovs_list *node;
1744 ovs_mutex_lock(&bond->mutex);
1745 if (ovs_list_is_empty(&bond->enabled_slaves)) {
1746 ovs_mutex_unlock(&bond->mutex);
1750 node = ovs_list_pop_front(&bond->enabled_slaves);
1751 ovs_list_push_back(&bond->enabled_slaves, node);
1752 ovs_mutex_unlock(&bond->mutex);
1754 return CONTAINER_OF(node, struct bond_slave, list_node);
1757 static struct bond_slave *
1758 choose_output_slave(const struct bond *bond, const struct flow *flow,
1759 struct flow_wildcards *wc, uint16_t vlan)
1761 struct bond_entry *e;
1764 balance = bond->balance;
1765 if (bond->lacp_status == LACP_CONFIGURED) {
1766 /* LACP has been configured on this bond but negotiations were
1767 * unsuccussful. If lacp_fallback_ab is enabled use active-
1768 * backup mode else drop all traffic. */
1769 if (!bond->lacp_fallback_ab) {
1777 return bond->active_slave;
1780 if (bond->lacp_status != LACP_NEGOTIATED) {
1781 /* Must have LACP negotiations for TCP balanced bonds. */
1785 flow_mask_hash_fields(flow, wc, NX_HASH_FIELDS_SYMMETRIC_L4);
1790 flow_mask_hash_fields(flow, wc, NX_HASH_FIELDS_ETH_SRC);
1792 e = lookup_bond_entry(bond, flow, vlan);
1793 if (!e->slave || !e->slave->enabled) {
1794 e->slave = get_enabled_slave(CONST_CAST(struct bond*, bond));
1803 static struct bond_slave *
1804 bond_choose_slave(const struct bond *bond)
1806 struct bond_slave *slave, *best;
1808 /* Find the last active slave. */
1809 slave = bond_find_slave_by_mac(bond, bond->active_slave_mac);
1810 if (slave && slave->enabled) {
1814 /* Find an enabled slave. */
1815 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1816 if (slave->enabled) {
1821 /* All interfaces are disabled. Find an interface that will be enabled
1822 * after its updelay expires. */
1824 HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
1825 if (slave->delay_expires != LLONG_MAX
1826 && slave->may_enable
1827 && (!best || slave->delay_expires < best->delay_expires)) {
1835 bond_choose_active_slave(struct bond *bond)
1837 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
1838 struct bond_slave *old_active_slave = bond->active_slave;
1840 bond->active_slave = bond_choose_slave(bond);
1841 if (bond->active_slave) {
1842 if (bond->active_slave->enabled) {
1843 VLOG_INFO_RL(&rl, "bond %s: active interface is now %s",
1844 bond->name, bond->active_slave->name);
1846 VLOG_INFO_RL(&rl, "bond %s: active interface is now %s, skipping "
1847 "remaining %lld ms updelay (since no interface was "
1848 "enabled)", bond->name, bond->active_slave->name,
1849 bond->active_slave->delay_expires - time_msec());
1850 bond_enable_slave(bond->active_slave, true);
1853 bond->send_learning_packets = true;
1855 if (bond->active_slave != old_active_slave) {
1856 bond_active_slave_changed(bond);
1858 } else if (old_active_slave) {
1859 VLOG_INFO_RL(&rl, "bond %s: all interfaces disabled", bond->name);
1864 * Return true if bond has unstored active slave change.
1865 * If return true, 'mac' will store the bond's current active slave's
1868 bond_get_changed_active_slave(const char *name, struct eth_addr *mac,
1873 ovs_rwlock_wrlock(&rwlock);
1874 bond = bond_find(name);
1876 if (bond->active_slave_changed || force) {
1877 *mac = bond->active_slave_mac;
1878 bond->active_slave_changed = false;
1879 ovs_rwlock_unlock(&rwlock);
1883 ovs_rwlock_unlock(&rwlock);