2 # Copyright (C) 2013 Nicira, Inc. All Rights Reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 # - Doesn't support multicast other than "unknown-dst"
30 import ovs.unixctl.server
38 __pychecker__ = 'no-reuseattr' # Remove in pychecker >= 0.8.19.
39 vlog = ovs.vlog.Vlog("ovs-vtep")
48 bfd_bridge = "vtep_bfd"
51 def call_prog(prog, args_list):
52 cmd = [prog, "-vconsole:off"] + args_list
53 output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
54 if len(output) == 0 or output[0] == None:
57 output = output[0].strip()
61 return call_prog("ovs-vsctl", shlex.split(args))
64 return call_prog("ovs-ofctl", shlex.split(args))
67 return call_prog("vtep-ctl", shlex.split(args))
70 def unixctl_exit(conn, unused_argv, unused_aux):
76 class Logical_Switch(object):
77 def __init__(self, ls_name):
81 self.short_name = "vtep_ls" + str(ls_count)
82 vlog.info("creating lswitch %s (%s)" % (self.name, self.short_name))
85 self.local_macs = set()
87 self.unknown_dsts = set()
92 vlog.info("destroying lswitch %s" % self.name)
95 column = vtep_ctl("--columns=tunnel_key find logical_switch "
96 "name=%s" % self.name)
97 tunnel_key = column.partition(":")[2].strip()
98 if (tunnel_key and type(eval(tunnel_key)) == types.IntType):
99 self.tunnel_key = tunnel_key
100 vlog.info("using tunnel key %s in %s"
101 % (self.tunnel_key, self.name))
104 vlog.warn("invalid tunnel key for %s, using 0" % self.name)
106 ovs_vsctl("--may-exist add-br %s" % self.short_name)
107 ovs_vsctl("br-set-external-id %s vtep_logical_switch true"
109 ovs_vsctl("br-set-external-id %s logical_switch_name %s"
110 % (self.short_name, self.name))
112 vtep_ctl("clear-local-macs %s" % self.name)
113 vtep_ctl("add-mcast-local %s unknown-dst %s" % (self.name, Tunnel_Ip))
115 ovs_ofctl("del-flows %s" % self.short_name)
116 ovs_ofctl("add-flow %s priority=0,action=drop" % self.short_name)
118 def cleanup_ls(self):
119 for port_no, tun_name, remote_ip in self.tunnels.itervalues():
122 def update_flood(self):
123 flood_ports = self.ports.values()
125 # Traffic flowing from one 'unknown-dst' should not be flooded to
126 # port belonging to another 'unknown-dst'.
127 for tunnel in self.unknown_dsts:
128 port_no = self.tunnels[tunnel][0]
129 ovs_ofctl("add-flow %s table=1,priority=1,in_port=%s,action=%s"
130 % (self.short_name, port_no, ",".join(flood_ports)))
132 # Traffic coming from a VTEP physical port should only be flooded to
133 # one 'unknown-dst' and to all other physical ports that belong to that
134 # VTEP device and this logical switch.
135 for tunnel in self.unknown_dsts:
136 port_no = self.tunnels[tunnel][0]
137 flood_ports.append(port_no)
140 ovs_ofctl("add-flow %s table=1,priority=0,action=%s"
141 % (self.short_name, ",".join(flood_ports)))
143 def add_lbinding(self, lbinding):
144 vlog.info("adding %s binding to %s" % (lbinding, self.name))
145 port_no = ovs_vsctl("get Interface %s ofport" % lbinding)
146 self.ports[lbinding] = port_no
147 ovs_ofctl("add-flow %s in_port=%s,action=learn(table=1,"
148 "priority=1000,idle_timeout=15,cookie=0x5000,"
149 "NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],"
150 "output:NXM_OF_IN_PORT[]),resubmit(,1)"
151 % (self.short_name, port_no))
155 def del_lbinding(self, lbinding):
156 vlog.info("removing %s binding from %s" % (lbinding, self.name))
157 port_no = self.ports[lbinding]
158 ovs_ofctl("del-flows %s in_port=%s" % (self.short_name, port_no));
159 del self.ports[lbinding]
162 def add_tunnel(self, tunnel):
164 vlog.info("adding tunnel %s" % tunnel)
165 encap, ip = tunnel.split("/")
167 if encap != "vxlan_over_ipv4":
168 vlog.warn("unsupported tunnel format %s" % encap)
172 tun_name = "vx" + str(tun_id)
174 ovs_vsctl("add-port %s %s -- set Interface %s type=vxlan "
175 "options:key=%s options:remote_ip=%s"
176 % (self.short_name, tun_name, tun_name, self.tunnel_key, ip))
179 port_no = ovs_vsctl("get Interface %s ofport" % tun_name)
183 vlog.warn("couldn't create tunnel %s" % tunnel)
184 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
187 # Give the system a moment to allocate the port number
190 self.tunnels[tunnel] = (port_no, tun_name, ip)
194 ovs_ofctl("add-flow %s table=0,priority=1000,in_port=%s,"
195 "actions=resubmit(,1)"
196 % (self.short_name, port_no))
198 def del_tunnel(self, tunnel):
199 vlog.info("removing tunnel %s" % tunnel)
201 port_no, tun_name, remote_ip = self.tunnels[tunnel]
202 ovs_ofctl("del-flows %s table=0,in_port=%s"
203 % (self.short_name, port_no))
204 ovs_vsctl("del-port %s %s" % (self.short_name, tun_name))
208 del self.tunnels[tunnel]
210 def update_local_macs(self):
211 flows = ovs_ofctl("dump-flows %s cookie=0x5000/-1,table=1"
212 % self.short_name).splitlines()
215 mac = re.split(r'.*dl_dst=(.*) .*', f)
219 for mac in macs.difference(self.local_macs):
220 vlog.info("adding local ucast %s to %s" % (mac, self.name))
221 vtep_ctl("add-ucast-local %s %s %s" % (self.name, mac, Tunnel_Ip))
223 for mac in self.local_macs.difference(macs):
224 vlog.info("removing local ucast %s from %s" % (mac, self.name))
225 vtep_ctl("del-ucast-local %s %s" % (self.name, mac))
227 self.local_macs = macs
229 def add_remote_mac(self, mac, tunnel):
230 port_no = self.tunnels.get(tunnel, (0,""))[0]
234 ovs_ofctl("add-flow %s table=1,priority=1000,dl_dst=%s,action=%s"
235 % (self.short_name, mac, port_no))
237 def del_remote_mac(self, mac):
238 ovs_ofctl("del-flows %s table=1,dl_dst=%s" % (self.short_name, mac))
240 def update_remote_macs(self):
246 mac_list = vtep_ctl("list-remote-macs %s" % self.name).splitlines()
247 for line in mac_list:
248 if (line.find("mcast-mac-remote") != -1):
252 entry = re.split(r' (.*) -> (.*)', line)
257 remote_macs[entry[1]] = entry[2]
259 if entry[1] != "unknown-dst":
262 unknown_dsts.add(entry[2])
264 tunnels.add(entry[2])
266 old_tunnels = set(self.tunnels.keys())
268 for tunnel in tunnels.difference(old_tunnels):
269 self.add_tunnel(tunnel)
271 for tunnel in old_tunnels.difference(tunnels):
272 self.del_tunnel(tunnel)
274 for mac in remote_macs.keys():
275 if (self.remote_macs.get(mac) != remote_macs[mac]):
276 self.add_remote_mac(mac, remote_macs[mac])
278 for mac in self.remote_macs.keys():
279 if not remote_macs.has_key(mac):
280 self.del_remote_mac(mac)
282 self.remote_macs = remote_macs
284 if (self.unknown_dsts != unknown_dsts):
285 self.unknown_dsts = unknown_dsts
288 def update_stats(self):
289 # Map Open_vSwitch's "interface:statistics" to columns of
290 # vtep's logical_binding_stats. Since we are using the 'interface' from
291 # the logical switch to collect stats, packets transmitted from it
292 # is received in the physical switch and vice versa.
293 stats_map = {'tx_packets':'packets_to_local',
294 'tx_bytes':'bytes_to_local',
295 'rx_packets':'packets_from_local',
296 'rx_bytes':'bytes_from_local'}
298 # Go through all the logical switch's interfaces that end with "-l"
299 # and copy the statistics to logical_binding_stats.
300 for interface in self.ports.iterkeys():
301 if not interface.endswith("-l"):
303 # Physical ports can have a '-' as part of its name.
304 vlan, remainder = interface.split("-", 1)
305 pp_name, logical = remainder.rsplit("-", 1)
306 uuid = vtep_ctl("get physical_port %s vlan_stats:%s"
311 for (mapfrom, mapto) in stats_map.iteritems():
312 value = ovs_vsctl("get interface %s statistics:%s"
313 % (interface, mapfrom)).strip('"')
314 vtep_ctl("set logical_binding_stats %s %s=%s"
315 % (uuid, mapto, value))
318 self.update_local_macs()
319 self.update_remote_macs()
322 def get_vtep_tunnel(remote_ip):
323 # Get the physical_locator record for the local tunnel end point.
324 column = vtep_ctl("--columns=_uuid find physical_locator "
325 "dst_ip=%s" % Tunnel_Ip)
326 local = column.partition(":")[2].strip()
328 return (None, None, None)
330 # Get the physical_locator record for the remote tunnel end point.
331 column = vtep_ctl("--columns=_uuid find physical_locator "
332 "dst_ip=%s" % remote_ip)
333 remote = column.partition(":")[2].strip()
335 return (None, None, None)
337 column = vtep_ctl("--columns=_uuid find tunnel "
338 "local=%s remote=%s" % (local, remote))
339 tunnel = column.partition(":")[2].strip()
341 return (local, remote, tunnel)
343 def create_vtep_tunnel(remote_ip):
344 local, remote, tunnel = get_vtep_tunnel(remote_ip)
345 if not local or not remote:
349 vlog.info("creating tunnel record in vtep for remote_ip:%s"
351 tunnel = vtep_ctl("add physical_switch %s tunnels @tun -- "
352 "--id=@tun create Tunnel local=%s remote=%s"
353 %(ps_name, local, remote))
356 def destroy_vtep_tunnel(remote_ip):
357 local, remote, tunnel = get_vtep_tunnel(remote_ip)
359 vlog.info("destroying tunnel record in vtep for remote_ip:%s"
361 vtep_ctl("remove physical_switch %s tunnels %s "
362 "-- --if-exists destroy tunnel %s"
363 % (ps_name, tunnel, tunnel))
365 def add_bfd(remote_ip):
366 # The VTEP emulator creates one OVS bridge for every logical switch.
367 # Multiple logical switches can have multiple OVS tunnels to the
368 # same machine (with different tunnel ids). But VTEP schema expects
369 # a single BFD session between two physical locators. Therefore
370 # create a separate bridge ('bfd_bridge') and create a single OVS tunnel
371 # between two phsyical locators (using reference counter).
372 if remote_ip in bfd_ref:
373 bfd_ref[remote_ip] += 1
376 vlog.info("adding bfd tunnel for remote_ip:%s" % remote_ip)
378 port_name = "bfd" + remote_ip
379 # Don't enable BFD yet. Enabling or disabling BFD is based on
380 # the controller setting a value in VTEP DB's tunnel record.
381 ovs_vsctl("--may-exist add-port %s %s "
382 " -- set Interface %s type=vxlan options:remote_ip=%s"
383 % (bfd_bridge, port_name, port_name, remote_ip))
384 bfd_ref[remote_ip] = 1
386 # Ideally, we should create a 'tunnel' record in the VTEP DB here.
387 # To create a 'tunnel' record, we need 2 entries in 'physical_locator'
388 # table (one for local and one for remote). But, 'physical_locator'
389 # can be created/destroyed asynchronously when the remote controller
390 # adds/removes entries in Ucast_Macs_Remote table. To prevent race
391 # conditions, pass the responsibility of creating a 'tunnel' record
392 # to run_bfd() which runs more often.
394 def del_bfd(remote_ip):
395 if remote_ip in bfd_ref:
396 if bfd_ref[remote_ip] == 1:
397 port_name = "bfd" + remote_ip
398 vlog.info("deleting bfd tunnel for remote_ip:%s" % remote_ip)
399 ovs_vsctl("--if-exists del-port %s" % port_name)
400 destroy_vtep_tunnel(remote_ip)
401 del bfd_ref[remote_ip]
403 bfd_ref[remote_ip] -= 1
406 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
407 for port in bfd_ports:
408 remote_ip = ovs_vsctl("get interface %s options:remote_ip" % port)
409 tunnel = create_vtep_tunnel(remote_ip)
413 bfd_params_default = {'bfd_params:enable' : 'false',
414 'bfd_params:min_rx' : 1000,
415 'bfd_params:min_tx' : 100,
416 'bfd_params:decay_min_rx' : 0,
417 'bfd_params:cpath_down' : 'false',
418 'bfd_params:check_tnl_key' : 'false'}
419 bfd_params_values = {}
421 for key, default in bfd_params_default.iteritems():
422 column = vtep_ctl("--if-exists get tunnel %s %s"
425 bfd_params_values[key] = default
427 bfd_params_values[key] = column
429 for key, value in bfd_params_values.iteritems():
430 new_key = key.replace('_params','')
431 ovs_vsctl("set interface %s %s=%s" % (port, new_key, value))
433 bfd_status = ['bfd_status:state', 'bfd_status:forwarding',
434 'bfd_status:diagnostic', 'bfd_status:remote_state',
435 'bfd_status:remote_diagnostic']
436 for key in bfd_status:
437 value = ovs_vsctl("--if-exists get interface %s %s" % (port, key))
439 vtep_ctl("set tunnel %s %s=%s" %(tunnel, key, value))
441 new_key = key.replace('bfd_status:', '')
442 vtep_ctl("remove tunnel %s bfd_status %s" % (tunnel, new_key))
444 vtep_ctl("set tunnel %s bfd_status:enabled=%s"
445 % (tunnel, bfd_params_values['bfd_params:enable']))
447 # Add the defaults as described in VTEP schema to make it explicit.
448 bfd_lconf_default = {'bfd_config_local:bfd_dst_ip' : '169.254.1.0',
449 'bfd_config_local:bfd_dst_mac' :
451 for key, value in bfd_lconf_default.iteritems():
452 vtep_ctl("set tunnel %s %s=%s" %(tunnel, key, value))
454 # bfd_config_remote options from VTEP DB should be populated to
455 # corresponding OVS DB values.
456 bfd_dst_ip = vtep_ctl("--if-exists get tunnel %s "
457 "bfd_config_remote:bfd_dst_ip" % (tunnel))
459 bfd_dst_ip = "169.254.1.1"
461 bfd_dst_mac = vtep_ctl("--if-exists get tunnel %s "
462 "bfd_config_remote:bfd_dst_mac" % (tunnel))
464 bfd_dst_mac = "00:23:20:00:00:01"
466 ovs_vsctl("set interface %s bfd:bfd_dst_ip=%s "
467 "bfd:bfd_remote_dst_mac=%s bfd:bfd_local_dst_mac=%s"
469 bfd_lconf_default['bfd_config_local:bfd_dst_mac'],
472 def add_binding(binding, ls):
473 vlog.info("adding binding %s" % binding)
475 vlan, pp_name = binding.split("-", 1)
476 pbinding = binding+"-p"
477 lbinding = binding+"-l"
479 # Create a patch port that connects the VLAN+port to the lswitch.
480 # Do them as two separate calls so if one side already exists, the
481 # other side is created.
482 ovs_vsctl("add-port %s %s "
483 " -- set Interface %s type=patch options:peer=%s"
484 % (ps_name, pbinding, pbinding, lbinding))
485 ovs_vsctl("add-port %s %s "
486 " -- set Interface %s type=patch options:peer=%s"
487 % (ls.short_name, lbinding, lbinding, pbinding))
489 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
490 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
491 vlan_ = vlan.lstrip('0')
493 ovs_ofctl("add-flow %s in_port=%s,dl_vlan=%s,action=strip_vlan,%s"
494 % (ps_name, port_no, vlan_, patch_no))
495 ovs_ofctl("add-flow %s in_port=%s,action=mod_vlan_vid:%s,%s"
496 % (ps_name, patch_no, vlan_, port_no))
498 ovs_ofctl("add-flow %s in_port=%s,action=%s"
499 % (ps_name, port_no, patch_no))
500 ovs_ofctl("add-flow %s in_port=%s,action=%s"
501 % (ps_name, patch_no, port_no))
503 # Create a logical_bindings_stats record.
506 vtep_ctl("set physical_port %s vlan_stats:%s=@stats --\
507 --id=@stats create logical_binding_stats packets_from_local=0"\
510 ls.add_lbinding(lbinding)
511 Bindings[binding] = ls.name
513 def del_binding(binding, ls):
514 vlog.info("removing binding %s" % binding)
516 vlan, pp_name = binding.split("-", 1)
517 pbinding = binding+"-p"
518 lbinding = binding+"-l"
520 port_no = ovs_vsctl("get Interface %s ofport" % pp_name)
521 patch_no = ovs_vsctl("get Interface %s ofport" % pbinding)
522 vlan_ = vlan.lstrip('0')
524 ovs_ofctl("del-flows %s in_port=%s,dl_vlan=%s"
525 % (ps_name, port_no, vlan_))
526 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
528 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, port_no))
529 ovs_ofctl("del-flows %s in_port=%s" % (ps_name, patch_no))
531 ls.del_lbinding(lbinding)
533 # Destroy the patch port that connects the VLAN+port to the lswitch
534 ovs_vsctl("del-port %s %s -- del-port %s %s"
535 % (ps_name, pbinding, ls.short_name, lbinding))
537 # Remove the record that links vlan with stats in logical_binding_stats.
538 vtep_ctl("remove physical_port %s vlan_stats %s" % (pp_name, vlan))
540 del Bindings[binding]
542 def handle_physical():
543 # Gather physical ports except the patch ports we created
544 ovs_ports = ovs_vsctl("list-ports %s" % ps_name).split()
545 ovs_port_set = set([port for port in ovs_ports if port[-2:] != "-p"])
547 vtep_pp_set = set(vtep_ctl("list-ports %s" % ps_name).split())
549 for pp_name in ovs_port_set.difference(vtep_pp_set):
550 vlog.info("adding %s to %s" % (pp_name, ps_name))
551 vtep_ctl("add-port %s %s" % (ps_name, pp_name))
553 for pp_name in vtep_pp_set.difference(ovs_port_set):
554 vlog.info("deleting %s from %s" % (pp_name, ps_name))
555 vtep_ctl("del-port %s %s" % (ps_name, pp_name))
558 for pp_name in vtep_pp_set:
559 binding_set = set(vtep_ctl("list-bindings %s %s"
560 % (ps_name, pp_name)).splitlines())
562 for b in binding_set:
563 vlan, ls_name = b.split()
564 if ls_name not in Lswitches:
565 Lswitches[ls_name] = Logical_Switch(ls_name)
567 binding = "%s-%s" % (vlan, pp_name)
568 ls = Lswitches[ls_name]
569 new_bindings.add(binding)
571 if Bindings.has_key(binding):
572 if Bindings[binding] == ls_name:
575 del_binding(binding, Lswitches[Bindings[binding]])
577 add_binding(binding, ls)
580 dead_bindings = set(Bindings.keys()).difference(new_bindings)
581 for binding in dead_bindings:
582 ls_name = Bindings[binding]
583 ls = Lswitches[ls_name]
585 del_binding(binding, ls)
587 if not len(ls.ports):
589 ovs_vsctl("del-br %s" % Lswitches[ls_name].short_name)
590 vtep_ctl("clear-local-macs %s" % Lswitches[ls_name].name)
591 del Lswitches[ls_name]
594 br_list = ovs_vsctl("list-br").split()
595 if (ps_name not in br_list):
596 ovs.util.ovs_fatal(0, "couldn't find OVS bridge %s" % ps_name, vlog)
598 call_prog("vtep-ctl", ["set", "physical_switch", ps_name,
599 'description="OVS VTEP Emulator"'])
601 tunnel_ips = vtep_ctl("get physical_switch %s tunnel_ips"
602 % ps_name).strip('[]"').split(", ")
603 if len(tunnel_ips) != 1 or not tunnel_ips[0]:
604 ovs.util.ovs_fatal(0, "exactly one 'tunnel_ips' should be set", vlog)
607 Tunnel_Ip = tunnel_ips[0]
609 ovs_ofctl("del-flows %s" % ps_name)
611 # Remove any logical bridges from the previous run
613 if ovs_vsctl("br-get-external-id %s vtep_logical_switch"
615 # Remove the remote side of any logical switch
616 ovs_ports = ovs_vsctl("list-ports %s" % br).split()
617 for port in ovs_ports:
618 port_type = ovs_vsctl("get Interface %s type"
620 if port_type != "patch":
623 peer = ovs_vsctl("get Interface %s options:peer"
626 ovs_vsctl("del-port %s" % peer)
628 ovs_vsctl("del-br %s" % br)
631 bfd_ports = ovs_vsctl("list-ports %s" % bfd_bridge).split()
632 for port in bfd_ports:
633 remote_ip = ovs_vsctl("get interface %s options:remote_ip"
635 tunnel = destroy_vtep_tunnel(remote_ip)
637 ovs_vsctl("del-br %s" % br)
639 ovs_vsctl("add-br %s" % bfd_bridge)
643 parser = argparse.ArgumentParser()
644 parser.add_argument("ps_name", metavar="PS-NAME",
645 help="Name of physical switch.")
646 parser.add_argument("--root-prefix", metavar="DIR",
647 help="Use DIR as alternate root directory"
649 parser.add_argument("--version", action="version",
650 version="%s %s" % (ovs.util.PROGRAM_NAME, VERSION))
652 ovs.vlog.add_args(parser)
653 ovs.daemon.add_args(parser)
654 args = parser.parse_args()
655 ovs.vlog.handle_args(args)
656 ovs.daemon.handle_args(args)
660 root_prefix = args.root_prefix
663 ps_name = args.ps_name
665 ovs.daemon.daemonize()
667 ovs.unixctl.command_register("exit", "", 0, 0, unixctl_exit, None)
668 error, unixctl = ovs.unixctl.server.UnixctlServer.create(None,
671 ovs.util.ovs_fatal(error, "could not create unixctl server", vlog)
682 for ls_name, ls in Lswitches.items():
687 poller = ovs.poller.Poller()
689 poller.timer_wait(1000)
694 if __name__ == '__main__':
698 # Let system.exit() calls complete normally
701 vlog.exception("traceback")
702 sys.exit(ovs.daemon.RESTART_EXIT_CODE)