2 # Copyright (C) 2013 Nicira, Inc. All Rights Reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 # - Doesn't support multicast other than "unknown-dst"
30 import ovs.unixctl.server
32 from six.moves import range
39 __pychecker__ = 'no-reuseattr' # Remove in pychecker >= 0.8.19.
40 vlog = ovs.vlog.Vlog("ovs-vtep")
50 bfd_bridge = "vtep_bfd"
54 def call_prog(prog, args_list):
55 cmd = [prog, "-vconsole:off"] + args_list
56 output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
57 if len(output) == 0 or output[0] is None:
60 output = output[0].strip()
65 return call_prog("ovs-vsctl", shlex.split(args))
69 return call_prog("ovs-ofctl", shlex.split(args))
73 return call_prog("vtep-ctl", shlex.split(args))
76 def unixctl_exit(conn, unused_argv, unused_aux):
82 class Logical_Switch(object):
83 def __init__(self, ls_name):
87 self.short_name = "vtep_ls" + str(ls_count)
88 vlog.info("creating lswitch %s (%s)" % (self.name, self.short_name))
91 self.local_macs = set()
93 self.unknown_dsts = set()
98 vlog.info("destroying lswitch %s" % self.name)
101 column = vtep_ctl("--columns=tunnel_key find logical_switch "
102 "name=%s" % self.name)
103 tunnel_key = column.partition(":")[2].strip()
104 if tunnel_key and isinstance(eval(tunnel_key), types.IntType):
105 self.tunnel_key = tunnel_key
106 vlog.info("using tunnel key %s in %s"
107 % (self.tunnel_key, self.name))
110 vlog.warn("invalid tunnel key for %s, using 0" % self.name)
113 ovs_vsctl("--may-exist add-br %s -- set Bridge %s datapath_type=%s"
114 % (self.short_name, self.short_name, ps_type))
116 ovs_vsctl("--may-exist add-br %s" % self.short_name)
118 ovs_vsctl("br-set-external-id %s vtep_logical_switch true"
120 ovs_vsctl("br-set-external-id %s logical_switch_name %s"
121 % (self.short_name, self.name))
123 vtep_ctl("clear-local-macs %s" % self.name)
124 vtep_ctl("add-mcast-local %s unknown-dst %s" % (self.name, Tunnel_Ip))
126 ovs_ofctl("del-flows %s" % self.short_name)
127 ovs_ofctl("add-flow %s priority=0,action=drop" % self.short_name)
129 def cleanup_ls(self):
130 for port_no, tun_name, remote_ip in self.tunnels.itervalues():
133 def update_flood(self):
134 flood_ports = self.ports.values()
136 # Traffic flowing from one 'unknown-dst' should not be flooded to
137 # port belonging to another 'unknown-dst'.
138 for tunnel in self.unknown_dsts:
139 port_no = self.tunnels[tunnel][0]
140 ovs_ofctl("add-flow %s table=1,priority=1,in_port=%s,action=%s"
141 % (self.short_name, port_no, ",".join(flood_ports)))
143 # Traffic coming from a VTEP physical port should only be flooded to
144 # one 'unknown-dst' and to all other physical ports that belong to that
145 # VTEP device and this logical switch.
146 for tunnel in self.unknown_dsts:
147 port_no = self.tunnels[tunnel][0]
148 flood_ports.append(port_no)
151 ovs_ofctl("add-flow %s table=1,priority=0,action=%s"
152 % (self.short_name, ",".join(flood_ports)))
154 def add_lbinding(self, lbinding):
155 vlog.info("adding %s binding to %s" % (lbinding, self.name))
156 port_no = ovs_vsctl("get Interface %s ofport" % lbinding)
157 self.ports[lbinding] = port_no
158 ovs_ofctl("add-flow %s in_port=%s,action=learn(table=1,"
159 "priority=1000,idle_timeout=15,cookie=0x5000,"
160 "NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],"
161 "output:NXM_OF_IN_PORT[]),resubmit(,1)"
162 % (self.short_name, port_no))
166 def del_lbinding(self, lbinding):
167 vlog.info("removing %s binding from %s" % (lbinding, self.name))
168 port_no = self.ports[lbinding]
169 ovs_ofctl("del-flows %s in_port=%s" % (self.short_name, port_no))
170 del self.ports[lbinding]
173 def add_tunnel(self, tunnel):
175 vlog.info("adding tunnel %s" % tunnel)
176 encap, ip = tunnel.split("/")
178 if encap != "vxlan_over_ipv4":
179 vlog.warn("unsupported tunnel format %s" % encap)
183 tun_name = "vx" + str(tun_id)
185 ovs_vsctl("add-port %s %s -- set Interface %s type=vxlan "
186 "options:key=%s options:remote_ip=%s"
187 % (self.short_name, tun_name, tun_name, self.tunnel_key, ip))
190 port_no = ovs_vsctl("get Interface %s ofport" % tun_name)
194 vlog.warn("couldn't create tunnel %s" % tunnel)
195 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
198 # Give the system a moment to allocate the port number
201 self.tunnels[tunnel] = (port_no, tun_name, ip)
205 ovs_ofctl("add-flow %s table=0,priority=1000,in_port=%s,"
206 "actions=resubmit(,1)"
207 % (self.short_name, port_no))
209 def del_tunnel(self, tunnel):
210 vlog.info("removing tunnel %s" % tunnel)
212 port_no, tun_name, remote_ip = self.tunnels[tunnel]
213 ovs_ofctl("del-flows %s table=0,in_port=%s"
214 % (self.short_name, port_no))
215 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
219 del self.tunnels[tunnel]
221 def update_local_macs(self):
222 flows = ovs_ofctl("dump-flows %s cookie=0x5000/-1,table=1"
223 % self.short_name).splitlines()
226 mac = re.split(r'.*dl_dst=(.*) .*', f)
230 for mac in macs.difference(self.local_macs):
231 vlog.info("adding local ucast %s to %s" % (mac, self.name))
232 vtep_ctl("add-ucast-local %s %s %s" % (self.name, mac, Tunnel_Ip))
234 for mac in self.local_macs.difference(macs):
235 vlog.info("removing local ucast %s from %s" % (mac, self.name))
236 vtep_ctl("del-ucast-local %s %s" % (self.name, mac))
238 self.local_macs = macs
240 def add_remote_mac(self, mac, tunnel):
241 port_no = self.tunnels.get(tunnel, (0, ""))[0]
245 ovs_ofctl("add-flow %s table=1,priority=1000,dl_dst=%s,action=%s"
246 % (self.short_name, mac, port_no))
248 def del_remote_mac(self, mac):
249 ovs_ofctl("del-flows %s table=1,dl_dst=%s" % (self.short_name, mac))
251 def update_remote_macs(self):
257 mac_list = vtep_ctl("list-remote-macs %s" % self.name).splitlines()
258 for line in mac_list:
259 if (line.find("mcast-mac-remote") != -1):
263 entry = re.split(r' (.*) -> (.*)', line)
268 remote_macs[entry[1]] = entry[2]
270 if entry[1] != "unknown-dst":
273 unknown_dsts.add(entry[2])
275 tunnels.add(entry[2])
277 old_tunnels = set(self.tunnels.keys())
279 for tunnel in tunnels.difference(old_tunnels):
280 self.add_tunnel(tunnel)
282 for tunnel in old_tunnels.difference(tunnels):
283 self.del_tunnel(tunnel)
285 for mac in remote_macs.keys():
286 if (self.remote_macs.get(mac) != remote_macs[mac]):
287 self.add_remote_mac(mac, remote_macs[mac])
289 for mac in self.remote_macs.keys():
290 if mac not in remote_macs:
291 self.del_remote_mac(mac)
293 self.remote_macs = remote_macs
295 if (self.unknown_dsts != unknown_dsts):
296 self.unknown_dsts = unknown_dsts
299 def update_stats(self):
300 # Map Open_vSwitch's "interface:statistics" to columns of
301 # vtep's logical_binding_stats. Since we are using the 'interface' from
302 # the logical switch to collect stats, packets transmitted from it
303 # is received in the physical switch and vice versa.
304 stats_map = {'tx_packets': 'packets_to_local',
305 'tx_bytes': 'bytes_to_local',
306 'rx_packets': 'packets_from_local',
307 'rx_bytes': 'bytes_from_local'}
309 # Go through all the logical switch's interfaces that end with "-l"
310 # and copy the statistics to logical_binding_stats.
311 for interface in self.ports.iterkeys():
312 if not interface.endswith("-l"):
314 # Physical ports can have a '-' as part of its name.
315 vlan, remainder = interface.split("-", 1)
316 pp_name, logical = remainder.rsplit("-", 1)
317 uuid = vtep_ctl("get physical_port %s vlan_stats:%s"
322 for (mapfrom, mapto) in stats_map.iteritems():
323 value = ovs_vsctl("get interface %s statistics:%s"
324 % (interface, mapfrom)).strip('"')
325 vtep_ctl("set logical_binding_stats %s %s=%s"
326 % (uuid, mapto, value))
329 self.update_local_macs()
330 self.update_remote_macs()
334 def get_vtep_tunnel(remote_ip):
335 # Get the physical_locator record for the local tunnel end point.
336 column = vtep_ctl("--columns=_uuid find physical_locator "
337 "dst_ip=%s" % Tunnel_Ip)
338 local = column.partition(":")[2].strip()
340 return (None, None, None)
342 # Get the physical_locator record for the remote tunnel end point.
343 column = vtep_ctl("--columns=_uuid find physical_locator "
344 "dst_ip=%s" % remote_ip)
345 remote = column.partition(":")[2].strip()
347 return (None, None, None)
349 column = vtep_ctl("--columns=_uuid find tunnel "
350 "local=%s remote=%s" % (local, remote))
351 tunnel = column.partition(":")[2].strip()
353 return (local, remote, tunnel)
356 def create_vtep_tunnel(remote_ip):
357 local, remote, tunnel = get_vtep_tunnel(remote_ip)
358 if not local or not remote:
362 vlog.info("creating tunnel record in vtep for remote_ip:%s"
364 tunnel = vtep_ctl("add physical_switch %s tunnels @tun -- "
365 "--id=@tun create Tunnel local=%s remote=%s"
366 % (ps_name, local, remote))
370 def destroy_vtep_tunnel(remote_ip):
371 local, remote, tunnel = get_vtep_tunnel(remote_ip)
373 vlog.info("destroying tunnel record in vtep for remote_ip:%s"
375 vtep_ctl("remove physical_switch %s tunnels %s "
376 "-- --if-exists destroy tunnel %s"
377 % (ps_name, tunnel, tunnel))
380 def add_bfd(remote_ip):
381 # The VTEP emulator creates one OVS bridge for every logical switch.
382 # Multiple logical switches can have multiple OVS tunnels to the
383 # same machine (with different tunnel ids). But VTEP schema expects
384 # a single BFD session between two physical locators. Therefore
385 # create a separate bridge ('bfd_bridge') and create a single OVS tunnel
386 # between two phsyical locators (using reference counter).
387 if remote_ip in bfd_ref:
388 bfd_ref[remote_ip] += 1
391 vlog.info("adding bfd tunnel for remote_ip:%s" % remote_ip)
393 port_name = "bfd" + remote_ip
394 # Don't enable BFD yet. Enabling or disabling BFD is based on
395 # the controller setting a value in VTEP DB's tunnel record.
396 ovs_vsctl("--may-exist add-port %s %s "
397 " -- set Interface %s type=vxlan options:remote_ip=%s"
398 % (bfd_bridge, port_name, port_name, remote_ip))
399 bfd_ref[remote_ip] = 1
401 # Ideally, we should create a 'tunnel' record in the VTEP DB here.
402 # To create a 'tunnel' record, we need 2 entries in 'physical_locator'
403 # table (one for local and one for remote). But, 'physical_locator'
404 # can be created/destroyed asynchronously when the remote controller
405 # adds/removes entries in Ucast_Macs_Remote table. To prevent race
406 # conditions, pass the responsibility of creating a 'tunnel' record
407 # to run_bfd() which runs more often.
410 def del_bfd(remote_ip):
411 if remote_ip in bfd_ref:
412 if bfd_ref[remote_ip] == 1:
413 port_name = "bfd" + remote_ip
414 vlog.info("deleting bfd tunnel for remote_ip:%s" % remote_ip)
415 ovs_vsctl("--if-exists del-port %s" % port_name)
416 destroy_vtep_tunnel(remote_ip)
417 del bfd_ref[remote_ip]
419 bfd_ref[remote_ip] -= 1
423 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
424 for port in bfd_ports:
425 remote_ip = ovs_vsctl("get interface %s options:remote_ip" % port)
426 tunnel = create_vtep_tunnel(remote_ip)
430 bfd_params_default = {'bfd_params:enable': 'false',
431 'bfd_params:min_rx': 1000,
432 'bfd_params:min_tx': 100,
433 'bfd_params:decay_min_rx': 0,
434 'bfd_params:cpath_down': 'false',
435 'bfd_params:check_tnl_key': 'false'}
436 bfd_params_values = {}
438 for key, default in bfd_params_default.iteritems():
439 column = vtep_ctl("--if-exists get tunnel %s %s"
442 bfd_params_values[key] = default
444 bfd_params_values[key] = column
446 for key, value in bfd_params_values.iteritems():
447 new_key = key.replace('_params', '')
448 ovs_vsctl("set interface %s %s=%s" % (port, new_key, value))
450 bfd_status = ['bfd_status:state', 'bfd_status:forwarding',
451 'bfd_status:diagnostic', 'bfd_status:remote_state',
452 'bfd_status:remote_diagnostic']
453 for key in bfd_status:
454 value = ovs_vsctl("--if-exists get interface %s %s" % (port, key))
456 vtep_ctl("set tunnel %s %s=%s" % (tunnel, key, value))
458 new_key = key.replace('bfd_status:', '')
459 vtep_ctl("remove tunnel %s bfd_status %s" % (tunnel, new_key))
461 vtep_ctl("set tunnel %s bfd_status:enabled=%s"
462 % (tunnel, bfd_params_values['bfd_params:enable']))
464 # Add the defaults as described in VTEP schema to make it explicit.
465 bfd_lconf_default = {'bfd_config_local:bfd_dst_ip': '169.254.1.0',
466 'bfd_config_local:bfd_dst_mac':
468 for key, value in bfd_lconf_default.iteritems():
469 vtep_ctl("set tunnel %s %s=%s" % (tunnel, key, value))
471 # bfd_config_remote options from VTEP DB should be populated to
472 # corresponding OVS DB values.
473 bfd_dst_ip = vtep_ctl("--if-exists get tunnel %s "
474 "bfd_config_remote:bfd_dst_ip" % (tunnel))
476 bfd_dst_ip = "169.254.1.1"
478 bfd_dst_mac = vtep_ctl("--if-exists get tunnel %s "
479 "bfd_config_remote:bfd_dst_mac" % (tunnel))
481 bfd_dst_mac = "00:23:20:00:00:01"
483 ovs_vsctl("set interface %s bfd:bfd_dst_ip=%s "
484 "bfd:bfd_remote_dst_mac=%s bfd:bfd_local_dst_mac=%s"
486 bfd_lconf_default['bfd_config_local:bfd_dst_mac'],
490 def add_binding(binding, ls):
491 vlog.info("adding binding %s" % binding)
493 vlan, pp_name = binding.split("-", 1)
494 pbinding = binding + "-p"
495 lbinding = binding + "-l"
497 # Create a patch port that connects the VLAN+port to the lswitch.
498 # Do them as two separate calls so if one side already exists, the
499 # other side is created.
500 ovs_vsctl("add-port %s %s "
501 " -- set Interface %s type=patch options:peer=%s"
502 % (ps_name, pbinding, pbinding, lbinding))
503 ovs_vsctl("add-port %s %s "
504 " -- set Interface %s type=patch options:peer=%s"
505 % (ls.short_name, lbinding, lbinding, pbinding))
507 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
508 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
509 vlan_ = vlan.lstrip('0')
511 ovs_ofctl("add-flow %s in_port=%s,dl_vlan=%s,action=strip_vlan,%s"
512 % (ps_name, port_no, vlan_, patch_no))
513 ovs_ofctl("add-flow %s in_port=%s,action=mod_vlan_vid:%s,%s"
514 % (ps_name, patch_no, vlan_, port_no))
516 ovs_ofctl("add-flow %s in_port=%s,action=%s"
517 % (ps_name, port_no, patch_no))
518 ovs_ofctl("add-flow %s in_port=%s,action=%s"
519 % (ps_name, patch_no, port_no))
521 # Create a logical_bindings_stats record.
524 vtep_ctl("set physical_port %s vlan_stats:%s=@stats -- "
525 "--id=@stats create logical_binding_stats packets_from_local=0"
528 ls.add_lbinding(lbinding)
529 Bindings[binding] = ls.name
532 def del_binding(binding, ls):
533 vlog.info("removing binding %s" % binding)
535 vlan, pp_name = binding.split("-", 1)
536 pbinding = binding + "-p"
537 lbinding = binding + "-l"
539 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
540 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
541 vlan_ = vlan.lstrip('0')
543 ovs_ofctl("del-flows %s in_port=%s,dl_vlan=%s"
544 % (ps_name, port_no, vlan_))
545 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
547 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, port_no))
548 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
550 ls.del_lbinding(lbinding)
552 # Destroy the patch port that connects the VLAN+port to the lswitch
553 ovs_vsctl("del-port %s %s -- del-port %s %s"
554 % (ps_name, pbinding, ls.short_name, lbinding))
556 # Remove the record that links vlan with stats in logical_binding_stats.
557 vtep_ctl("remove physical_port %s vlan_stats %s" % (pp_name, vlan))
559 del Bindings[binding]
562 def handle_physical():
563 # Gather physical ports except the patch ports we created
564 ovs_ports = ovs_vsctl("list-ports %s" % ps_name).split()
565 ovs_port_set = set([port for port in ovs_ports if port[-2:] != "-p"])
567 vtep_pp_set = set(vtep_ctl("list-ports %s" % ps_name).split())
569 for pp_name in ovs_port_set.difference(vtep_pp_set):
570 vlog.info("adding %s to %s" % (pp_name, ps_name))
571 vtep_ctl("add-port %s %s" % (ps_name, pp_name))
573 for pp_name in vtep_pp_set.difference(ovs_port_set):
574 vlog.info("deleting %s from %s" % (pp_name, ps_name))
575 vtep_ctl("del-port %s %s" % (ps_name, pp_name))
578 for pp_name in vtep_pp_set:
579 binding_set = set(vtep_ctl("list-bindings %s %s"
580 % (ps_name, pp_name)).splitlines())
582 for b in binding_set:
583 vlan, ls_name = b.split()
584 if ls_name not in Lswitches:
585 Lswitches[ls_name] = Logical_Switch(ls_name)
587 binding = "%s-%s" % (vlan, pp_name)
588 ls = Lswitches[ls_name]
589 new_bindings.add(binding)
591 if binding in Bindings:
592 if Bindings[binding] == ls_name:
595 del_binding(binding, Lswitches[Bindings[binding]])
597 add_binding(binding, ls)
599 dead_bindings = set(Bindings.keys()).difference(new_bindings)
600 for binding in dead_bindings:
601 ls_name = Bindings[binding]
602 ls = Lswitches[ls_name]
604 del_binding(binding, ls)
606 if not len(ls.ports):
608 ovs_vsctl("del-br %s" % Lswitches[ls_name].short_name)
609 vtep_ctl("clear-local-macs %s" % Lswitches[ls_name].name)
610 del Lswitches[ls_name]
614 br_list = ovs_vsctl("list-br").split()
615 if (ps_name not in br_list):
616 ovs.util.ovs_fatal(0, "couldn't find OVS bridge %s" % ps_name, vlog)
619 ps_type = ovs_vsctl("get Bridge %s datapath_type" % ps_name).strip('"')
621 call_prog("vtep-ctl", ["set", "physical_switch", ps_name,
622 'description="OVS VTEP Emulator"'])
624 tunnel_ips = vtep_ctl("get physical_switch %s tunnel_ips"
625 % ps_name).strip('[]"').split(", ")
626 if len(tunnel_ips) != 1 or not tunnel_ips[0]:
627 ovs.util.ovs_fatal(0, "exactly one 'tunnel_ips' should be set", vlog)
630 Tunnel_Ip = tunnel_ips[0]
632 ovs_ofctl("del-flows %s" % ps_name)
634 # Remove any logical bridges from the previous run
636 if ovs_vsctl("br-get-external-id %s vtep_logical_switch"
638 # Remove the remote side of any logical switch
639 ovs_ports = ovs_vsctl("list-ports %s" % br).split()
640 for port in ovs_ports:
641 port_type = ovs_vsctl("get Interface %s type"
643 if port_type != "patch":
646 peer = ovs_vsctl("get Interface %s options:peer"
649 ovs_vsctl("del-port %s" % peer)
651 ovs_vsctl("del-br %s" % br)
654 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
655 for port in bfd_ports:
656 remote_ip = ovs_vsctl("get interface %s options:remote_ip"
658 destroy_vtep_tunnel(remote_ip)
660 ovs_vsctl("del-br %s" % br)
663 ovs_vsctl("add-br %s -- set Bridge %s datapath_type=%s"
664 % (bfd_bridge, bfd_bridge, ps_type))
666 ovs_vsctl("add-br %s" % bfd_bridge)
668 # Remove local-mac entries from the previous run. Otherwise, if a vlan
669 # binding is removed while the emulator is *not* running, the corresponding
670 # local-mac entries are never cleaned up.
671 vtep_ls = set(vtep_ctl("list-ls").split())
672 for ls_name in vtep_ls:
673 vtep_ctl("clear-local-macs %s" % ls_name)
677 parser = argparse.ArgumentParser()
678 parser.add_argument("ps_name", metavar="PS-NAME",
679 help="Name of physical switch.")
680 parser.add_argument("--root-prefix", metavar="DIR",
681 help="Use DIR as alternate root directory"
683 parser.add_argument("--version", action="version",
684 version="%s %s" % (ovs.util.PROGRAM_NAME, VERSION))
686 ovs.vlog.add_args(parser)
687 ovs.daemon.add_args(parser)
688 args = parser.parse_args()
689 ovs.vlog.handle_args(args)
690 ovs.daemon.handle_args(args)
694 root_prefix = args.root_prefix
697 ps_name = args.ps_name
699 ovs.daemon.daemonize()
701 ovs.unixctl.command_register("exit", "", 0, 0, unixctl_exit, None)
702 error, unixctl = ovs.unixctl.server.UnixctlServer.create(None,
705 ovs.util.ovs_fatal(error, "could not create unixctl server", vlog)
716 for ls_name, ls in Lswitches.items():
721 poller = ovs.poller.Poller()
723 poller.timer_wait(1000)
728 if __name__ == '__main__':
732 # Let system.exit() calls complete normally
735 vlog.exception("traceback")
736 sys.exit(ovs.daemon.RESTART_EXIT_CODE)