2 # Copyright (C) 2013 Nicira, Inc. All Rights Reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 # - Doesn't support multicast other than "unknown-dst"
30 import ovs.unixctl.server
32 from six.moves import range
40 __pychecker__ = 'no-reuseattr' # Remove in pychecker >= 0.8.19.
41 vlog = ovs.vlog.Vlog("ovs-vtep")
51 bfd_bridge = "vtep_bfd"
55 def call_prog(prog, args_list):
56 cmd = [prog, "-vconsole:off"] + args_list
57 output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
58 if len(output) == 0 or output[0] is None:
61 output = output[0].strip()
66 return call_prog("ovs-vsctl", shlex.split(args))
70 return call_prog("ovs-ofctl", shlex.split(args))
74 return call_prog("vtep-ctl", shlex.split(args))
77 def unixctl_exit(conn, unused_argv, unused_aux):
83 class Logical_Switch(object):
84 def __init__(self, ls_name):
88 self.short_name = "vtep_ls" + str(ls_count)
89 vlog.info("creating lswitch %s (%s)" % (self.name, self.short_name))
92 self.local_macs = set()
94 self.unknown_dsts = set()
99 vlog.info("destroying lswitch %s" % self.name)
102 column = vtep_ctl("--columns=tunnel_key find logical_switch "
103 "name=%s" % self.name)
104 tunnel_key = column.partition(":")[2].strip()
105 if tunnel_key and isinstance(eval(tunnel_key), types.IntType):
106 self.tunnel_key = tunnel_key
107 vlog.info("using tunnel key %s in %s"
108 % (self.tunnel_key, self.name))
111 vlog.warn("invalid tunnel key for %s, using 0" % self.name)
114 ovs_vsctl("--may-exist add-br %s -- set Bridge %s datapath_type=%s"
115 % (self.short_name, self.short_name, ps_type))
117 ovs_vsctl("--may-exist add-br %s" % self.short_name)
119 ovs_vsctl("br-set-external-id %s vtep_logical_switch true"
121 ovs_vsctl("br-set-external-id %s logical_switch_name %s"
122 % (self.short_name, self.name))
124 vtep_ctl("clear-local-macs %s" % self.name)
125 vtep_ctl("add-mcast-local %s unknown-dst %s" % (self.name, Tunnel_Ip))
127 ovs_ofctl("del-flows %s" % self.short_name)
128 ovs_ofctl("add-flow %s priority=0,action=drop" % self.short_name)
130 def cleanup_ls(self):
131 for port_no, tun_name, remote_ip in six.itervalues(self.tunnels):
134 def update_flood(self):
135 flood_ports = list(self.ports.values())
137 # Traffic flowing from one 'unknown-dst' should not be flooded to
138 # port belonging to another 'unknown-dst'.
139 for tunnel in self.unknown_dsts:
140 port_no = self.tunnels[tunnel][0]
141 ovs_ofctl("add-flow %s table=1,priority=1,in_port=%s,action=%s"
142 % (self.short_name, port_no, ",".join(flood_ports)))
144 # Traffic coming from a VTEP physical port should only be flooded to
145 # one 'unknown-dst' and to all other physical ports that belong to that
146 # VTEP device and this logical switch.
147 for tunnel in self.unknown_dsts:
148 port_no = self.tunnels[tunnel][0]
149 flood_ports.append(port_no)
152 ovs_ofctl("add-flow %s table=1,priority=0,action=%s"
153 % (self.short_name, ",".join(flood_ports)))
155 def add_lbinding(self, lbinding):
156 vlog.info("adding %s binding to %s" % (lbinding, self.name))
157 port_no = ovs_vsctl("get Interface %s ofport" % lbinding)
158 self.ports[lbinding] = port_no
159 ovs_ofctl("add-flow %s in_port=%s,action=learn(table=1,"
160 "priority=1000,idle_timeout=15,cookie=0x5000,"
161 "NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],"
162 "output:NXM_OF_IN_PORT[]),resubmit(,1)"
163 % (self.short_name, port_no))
167 def del_lbinding(self, lbinding):
168 vlog.info("removing %s binding from %s" % (lbinding, self.name))
169 port_no = self.ports[lbinding]
170 ovs_ofctl("del-flows %s in_port=%s" % (self.short_name, port_no))
171 del self.ports[lbinding]
174 def add_tunnel(self, tunnel):
176 vlog.info("adding tunnel %s" % tunnel)
177 encap, ip = tunnel.split("/")
179 if encap != "vxlan_over_ipv4":
180 vlog.warn("unsupported tunnel format %s" % encap)
184 tun_name = "vx" + str(tun_id)
186 ovs_vsctl("add-port %s %s -- set Interface %s type=vxlan "
187 "options:key=%s options:remote_ip=%s"
188 % (self.short_name, tun_name, tun_name, self.tunnel_key, ip))
191 port_no = ovs_vsctl("get Interface %s ofport" % tun_name)
195 vlog.warn("couldn't create tunnel %s" % tunnel)
196 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
199 # Give the system a moment to allocate the port number
202 self.tunnels[tunnel] = (port_no, tun_name, ip)
206 ovs_ofctl("add-flow %s table=0,priority=1000,in_port=%s,"
207 "actions=resubmit(,1)"
208 % (self.short_name, port_no))
210 def del_tunnel(self, tunnel):
211 vlog.info("removing tunnel %s" % tunnel)
213 port_no, tun_name, remote_ip = self.tunnels[tunnel]
214 ovs_ofctl("del-flows %s table=0,in_port=%s"
215 % (self.short_name, port_no))
216 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
220 del self.tunnels[tunnel]
222 def update_local_macs(self):
223 flows = ovs_ofctl("dump-flows %s cookie=0x5000/-1,table=1"
224 % self.short_name).splitlines()
227 mac = re.split(r'.*dl_dst=(.*) .*', f)
231 for mac in macs.difference(self.local_macs):
232 vlog.info("adding local ucast %s to %s" % (mac, self.name))
233 vtep_ctl("add-ucast-local %s %s %s" % (self.name, mac, Tunnel_Ip))
235 for mac in self.local_macs.difference(macs):
236 vlog.info("removing local ucast %s from %s" % (mac, self.name))
237 vtep_ctl("del-ucast-local %s %s" % (self.name, mac))
239 self.local_macs = macs
241 def add_remote_mac(self, mac, tunnel):
242 port_no = self.tunnels.get(tunnel, (0, ""))[0]
246 ovs_ofctl("add-flow %s table=1,priority=1000,dl_dst=%s,action=%s"
247 % (self.short_name, mac, port_no))
249 def del_remote_mac(self, mac):
250 ovs_ofctl("del-flows %s table=1,dl_dst=%s" % (self.short_name, mac))
252 def update_remote_macs(self):
258 mac_list = vtep_ctl("list-remote-macs %s" % self.name).splitlines()
259 for line in mac_list:
260 if (line.find("mcast-mac-remote") != -1):
264 entry = re.split(r' (.*) -> (.*)', line)
269 remote_macs[entry[1]] = entry[2]
271 if entry[1] != "unknown-dst":
274 unknown_dsts.add(entry[2])
276 tunnels.add(entry[2])
278 old_tunnels = set(self.tunnels.keys())
280 for tunnel in tunnels.difference(old_tunnels):
281 self.add_tunnel(tunnel)
283 for tunnel in old_tunnels.difference(tunnels):
284 self.del_tunnel(tunnel)
286 for mac in six.iterkeys(remote_macs):
287 if (self.remote_macs.get(mac) != remote_macs[mac]):
288 self.add_remote_mac(mac, remote_macs[mac])
290 for mac in six.iterkeys(self.remote_macs):
291 if mac not in remote_macs:
292 self.del_remote_mac(mac)
294 self.remote_macs = remote_macs
296 if (self.unknown_dsts != unknown_dsts):
297 self.unknown_dsts = unknown_dsts
300 def update_stats(self):
301 # Map Open_vSwitch's "interface:statistics" to columns of
302 # vtep's logical_binding_stats. Since we are using the 'interface' from
303 # the logical switch to collect stats, packets transmitted from it
304 # is received in the physical switch and vice versa.
305 stats_map = {'tx_packets': 'packets_to_local',
306 'tx_bytes': 'bytes_to_local',
307 'rx_packets': 'packets_from_local',
308 'rx_bytes': 'bytes_from_local'}
310 # Go through all the logical switch's interfaces that end with "-l"
311 # and copy the statistics to logical_binding_stats.
312 for interface in six.iterkeys(self.ports):
313 if not interface.endswith("-l"):
315 # Physical ports can have a '-' as part of its name.
316 vlan, remainder = interface.split("-", 1)
317 pp_name, logical = remainder.rsplit("-", 1)
318 uuid = vtep_ctl("get physical_port %s vlan_stats:%s"
323 for mapfrom, mapto in six.iteritems(stats_map):
324 value = ovs_vsctl("get interface %s statistics:%s"
325 % (interface, mapfrom)).strip('"')
326 vtep_ctl("set logical_binding_stats %s %s=%s"
327 % (uuid, mapto, value))
330 self.update_local_macs()
331 self.update_remote_macs()
335 def get_vtep_tunnel(remote_ip):
336 # Get the physical_locator record for the local tunnel end point.
337 column = vtep_ctl("--columns=_uuid find physical_locator "
338 "dst_ip=%s" % Tunnel_Ip)
339 local = column.partition(":")[2].strip()
341 return (None, None, None)
343 # Get the physical_locator record for the remote tunnel end point.
344 column = vtep_ctl("--columns=_uuid find physical_locator "
345 "dst_ip=%s" % remote_ip)
346 remote = column.partition(":")[2].strip()
348 return (None, None, None)
350 column = vtep_ctl("--columns=_uuid find tunnel "
351 "local=%s remote=%s" % (local, remote))
352 tunnel = column.partition(":")[2].strip()
354 return (local, remote, tunnel)
357 def create_vtep_tunnel(remote_ip):
358 local, remote, tunnel = get_vtep_tunnel(remote_ip)
359 if not local or not remote:
363 vlog.info("creating tunnel record in vtep for remote_ip:%s"
365 tunnel = vtep_ctl("add physical_switch %s tunnels @tun -- "
366 "--id=@tun create Tunnel local=%s remote=%s"
367 % (ps_name, local, remote))
371 def destroy_vtep_tunnel(remote_ip):
372 local, remote, tunnel = get_vtep_tunnel(remote_ip)
374 vlog.info("destroying tunnel record in vtep for remote_ip:%s"
376 vtep_ctl("remove physical_switch %s tunnels %s "
377 "-- --if-exists destroy tunnel %s"
378 % (ps_name, tunnel, tunnel))
381 def add_bfd(remote_ip):
382 # The VTEP emulator creates one OVS bridge for every logical switch.
383 # Multiple logical switches can have multiple OVS tunnels to the
384 # same machine (with different tunnel ids). But VTEP schema expects
385 # a single BFD session between two physical locators. Therefore
386 # create a separate bridge ('bfd_bridge') and create a single OVS tunnel
387 # between two phsyical locators (using reference counter).
388 if remote_ip in bfd_ref:
389 bfd_ref[remote_ip] += 1
392 vlog.info("adding bfd tunnel for remote_ip:%s" % remote_ip)
394 port_name = "bfd" + remote_ip
395 # Don't enable BFD yet. Enabling or disabling BFD is based on
396 # the controller setting a value in VTEP DB's tunnel record.
397 ovs_vsctl("--may-exist add-port %s %s "
398 " -- set Interface %s type=vxlan options:remote_ip=%s"
399 % (bfd_bridge, port_name, port_name, remote_ip))
400 bfd_ref[remote_ip] = 1
402 # Ideally, we should create a 'tunnel' record in the VTEP DB here.
403 # To create a 'tunnel' record, we need 2 entries in 'physical_locator'
404 # table (one for local and one for remote). But, 'physical_locator'
405 # can be created/destroyed asynchronously when the remote controller
406 # adds/removes entries in Ucast_Macs_Remote table. To prevent race
407 # conditions, pass the responsibility of creating a 'tunnel' record
408 # to run_bfd() which runs more often.
411 def del_bfd(remote_ip):
412 if remote_ip in bfd_ref:
413 if bfd_ref[remote_ip] == 1:
414 port_name = "bfd" + remote_ip
415 vlog.info("deleting bfd tunnel for remote_ip:%s" % remote_ip)
416 ovs_vsctl("--if-exists del-port %s" % port_name)
417 destroy_vtep_tunnel(remote_ip)
418 del bfd_ref[remote_ip]
420 bfd_ref[remote_ip] -= 1
424 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
425 for port in bfd_ports:
426 remote_ip = ovs_vsctl("get interface %s options:remote_ip" % port)
427 tunnel = create_vtep_tunnel(remote_ip)
431 bfd_params_default = {'bfd_params:enable': 'false',
432 'bfd_params:min_rx': 1000,
433 'bfd_params:min_tx': 100,
434 'bfd_params:decay_min_rx': 0,
435 'bfd_params:cpath_down': 'false',
436 'bfd_params:check_tnl_key': 'false'}
437 bfd_params_values = {}
439 for key, default in six.iteritems(bfd_params_default):
440 column = vtep_ctl("--if-exists get tunnel %s %s"
443 bfd_params_values[key] = default
445 bfd_params_values[key] = column
447 for key, value in six.iteritems(bfd_params_values):
448 new_key = key.replace('_params', '')
449 ovs_vsctl("set interface %s %s=%s" % (port, new_key, value))
451 bfd_status = ['bfd_status:state', 'bfd_status:forwarding',
452 'bfd_status:diagnostic', 'bfd_status:remote_state',
453 'bfd_status:remote_diagnostic']
454 for key in bfd_status:
455 value = ovs_vsctl("--if-exists get interface %s %s" % (port, key))
457 vtep_ctl("set tunnel %s %s=%s" % (tunnel, key, value))
459 new_key = key.replace('bfd_status:', '')
460 vtep_ctl("remove tunnel %s bfd_status %s" % (tunnel, new_key))
462 vtep_ctl("set tunnel %s bfd_status:enabled=%s"
463 % (tunnel, bfd_params_values['bfd_params:enable']))
465 # Add the defaults as described in VTEP schema to make it explicit.
466 bfd_lconf_default = {'bfd_config_local:bfd_dst_ip': '169.254.1.0',
467 'bfd_config_local:bfd_dst_mac':
469 for key, value in six.iteritems(bfd_lconf_default):
470 vtep_ctl("set tunnel %s %s=%s" % (tunnel, key, value))
472 # bfd_config_remote options from VTEP DB should be populated to
473 # corresponding OVS DB values.
474 bfd_dst_ip = vtep_ctl("--if-exists get tunnel %s "
475 "bfd_config_remote:bfd_dst_ip" % (tunnel))
477 bfd_dst_ip = "169.254.1.1"
479 bfd_dst_mac = vtep_ctl("--if-exists get tunnel %s "
480 "bfd_config_remote:bfd_dst_mac" % (tunnel))
482 bfd_dst_mac = "00:23:20:00:00:01"
484 ovs_vsctl("set interface %s bfd:bfd_dst_ip=%s "
485 "bfd:bfd_remote_dst_mac=%s bfd:bfd_local_dst_mac=%s"
487 bfd_lconf_default['bfd_config_local:bfd_dst_mac'],
491 def add_binding(binding, ls):
492 vlog.info("adding binding %s" % binding)
494 vlan, pp_name = binding.split("-", 1)
495 pbinding = binding + "-p"
496 lbinding = binding + "-l"
498 # Create a patch port that connects the VLAN+port to the lswitch.
499 # Do them as two separate calls so if one side already exists, the
500 # other side is created.
501 ovs_vsctl("add-port %s %s "
502 " -- set Interface %s type=patch options:peer=%s"
503 % (ps_name, pbinding, pbinding, lbinding))
504 ovs_vsctl("add-port %s %s "
505 " -- set Interface %s type=patch options:peer=%s"
506 % (ls.short_name, lbinding, lbinding, pbinding))
508 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
509 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
510 vlan_ = vlan.lstrip('0')
512 ovs_ofctl("add-flow %s in_port=%s,dl_vlan=%s,action=strip_vlan,%s"
513 % (ps_name, port_no, vlan_, patch_no))
514 ovs_ofctl("add-flow %s in_port=%s,action=mod_vlan_vid:%s,%s"
515 % (ps_name, patch_no, vlan_, port_no))
517 ovs_ofctl("add-flow %s in_port=%s,action=%s"
518 % (ps_name, port_no, patch_no))
519 ovs_ofctl("add-flow %s in_port=%s,action=%s"
520 % (ps_name, patch_no, port_no))
522 # Create a logical_bindings_stats record.
525 vtep_ctl("set physical_port %s vlan_stats:%s=@stats -- "
526 "--id=@stats create logical_binding_stats packets_from_local=0"
529 ls.add_lbinding(lbinding)
530 Bindings[binding] = ls.name
533 def del_binding(binding, ls):
534 vlog.info("removing binding %s" % binding)
536 vlan, pp_name = binding.split("-", 1)
537 pbinding = binding + "-p"
538 lbinding = binding + "-l"
540 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
541 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
542 vlan_ = vlan.lstrip('0')
544 ovs_ofctl("del-flows %s in_port=%s,dl_vlan=%s"
545 % (ps_name, port_no, vlan_))
546 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
548 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, port_no))
549 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
551 ls.del_lbinding(lbinding)
553 # Destroy the patch port that connects the VLAN+port to the lswitch
554 ovs_vsctl("del-port %s %s -- del-port %s %s"
555 % (ps_name, pbinding, ls.short_name, lbinding))
557 # Remove the record that links vlan with stats in logical_binding_stats.
558 vtep_ctl("remove physical_port %s vlan_stats %s" % (pp_name, vlan))
560 del Bindings[binding]
563 def handle_physical():
564 # Gather physical ports except the patch ports we created
565 ovs_ports = ovs_vsctl("list-ports %s" % ps_name).split()
566 ovs_port_set = set([port for port in ovs_ports if port[-2:] != "-p"])
568 vtep_pp_set = set(vtep_ctl("list-ports %s" % ps_name).split())
570 for pp_name in ovs_port_set.difference(vtep_pp_set):
571 vlog.info("adding %s to %s" % (pp_name, ps_name))
572 vtep_ctl("add-port %s %s" % (ps_name, pp_name))
574 for pp_name in vtep_pp_set.difference(ovs_port_set):
575 vlog.info("deleting %s from %s" % (pp_name, ps_name))
576 vtep_ctl("del-port %s %s" % (ps_name, pp_name))
579 for pp_name in vtep_pp_set:
580 binding_set = set(vtep_ctl("list-bindings %s %s"
581 % (ps_name, pp_name)).splitlines())
583 for b in binding_set:
584 vlan, ls_name = b.split()
585 if ls_name not in Lswitches:
586 Lswitches[ls_name] = Logical_Switch(ls_name)
588 binding = "%s-%s" % (vlan, pp_name)
589 ls = Lswitches[ls_name]
590 new_bindings.add(binding)
592 if binding in Bindings:
593 if Bindings[binding] == ls_name:
596 del_binding(binding, Lswitches[Bindings[binding]])
598 add_binding(binding, ls)
600 dead_bindings = set(Bindings.keys()).difference(new_bindings)
601 for binding in dead_bindings:
602 ls_name = Bindings[binding]
603 ls = Lswitches[ls_name]
605 del_binding(binding, ls)
607 if not len(ls.ports):
609 ovs_vsctl("del-br %s" % Lswitches[ls_name].short_name)
610 vtep_ctl("clear-local-macs %s" % Lswitches[ls_name].name)
611 del Lswitches[ls_name]
615 br_list = ovs_vsctl("list-br").split()
616 if (ps_name not in br_list):
617 ovs.util.ovs_fatal(0, "couldn't find OVS bridge %s" % ps_name, vlog)
620 ps_type = ovs_vsctl("get Bridge %s datapath_type" % ps_name).strip('"')
622 call_prog("vtep-ctl", ["set", "physical_switch", ps_name,
623 'description="OVS VTEP Emulator"'])
625 tunnel_ips = vtep_ctl("get physical_switch %s tunnel_ips"
626 % ps_name).strip('[]"').split(", ")
627 if len(tunnel_ips) != 1 or not tunnel_ips[0]:
628 ovs.util.ovs_fatal(0, "exactly one 'tunnel_ips' should be set", vlog)
631 Tunnel_Ip = tunnel_ips[0]
633 ovs_ofctl("del-flows %s" % ps_name)
635 # Remove any logical bridges from the previous run
637 if ovs_vsctl("br-get-external-id %s vtep_logical_switch"
639 # Remove the remote side of any logical switch
640 ovs_ports = ovs_vsctl("list-ports %s" % br).split()
641 for port in ovs_ports:
642 port_type = ovs_vsctl("get Interface %s type"
644 if port_type != "patch":
647 peer = ovs_vsctl("get Interface %s options:peer"
650 ovs_vsctl("del-port %s" % peer)
652 ovs_vsctl("del-br %s" % br)
655 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
656 for port in bfd_ports:
657 remote_ip = ovs_vsctl("get interface %s options:remote_ip"
659 destroy_vtep_tunnel(remote_ip)
661 ovs_vsctl("del-br %s" % br)
664 ovs_vsctl("add-br %s -- set Bridge %s datapath_type=%s"
665 % (bfd_bridge, bfd_bridge, ps_type))
667 ovs_vsctl("add-br %s" % bfd_bridge)
669 # Remove local-mac entries from the previous run. Otherwise, if a vlan
670 # binding is removed while the emulator is *not* running, the corresponding
671 # local-mac entries are never cleaned up.
672 vtep_ls = set(vtep_ctl("list-ls").split())
673 for ls_name in vtep_ls:
674 vtep_ctl("clear-local-macs %s" % ls_name)
678 parser = argparse.ArgumentParser()
679 parser.add_argument("ps_name", metavar="PS-NAME",
680 help="Name of physical switch.")
681 parser.add_argument("--root-prefix", metavar="DIR",
682 help="Use DIR as alternate root directory"
684 parser.add_argument("--version", action="version",
685 version="%s %s" % (ovs.util.PROGRAM_NAME, VERSION))
687 ovs.vlog.add_args(parser)
688 ovs.daemon.add_args(parser)
689 args = parser.parse_args()
690 ovs.vlog.handle_args(args)
691 ovs.daemon.handle_args(args)
695 root_prefix = args.root_prefix
698 ps_name = args.ps_name
700 ovs.daemon.daemonize()
702 ovs.unixctl.command_register("exit", "", 0, 0, unixctl_exit, None)
703 error, unixctl = ovs.unixctl.server.UnixctlServer.create(None,
706 ovs.util.ovs_fatal(error, "could not create unixctl server", vlog)
717 for ls_name, ls in six.iteritems(Lswitches):
722 poller = ovs.poller.Poller()
724 poller.timer_wait(1000)
729 if __name__ == '__main__':
733 # Let system.exit() calls complete normally
736 vlog.exception("traceback")
737 sys.exit(ovs.daemon.RESTART_EXIT_CODE)