netdev-dpdk: fix mbuf leaks
[cascardo/ovs.git] / utilities / ovs-dev.py
index 21f0fc5..c121706 100755 (executable)
@@ -1,5 +1,5 @@
-#!/usr/bin/python
-# Copyright (c) 2013 Nicira, Inc.
+#! /usr/bin/env python
+# Copyright (c) 2013, 2014, 2015 Nicira, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,17 +22,22 @@ import tempfile
 
 ENV = os.environ
 HOME = ENV["HOME"]
+PWD = os.getcwd()
 OVS_SRC = HOME + "/ovs"
-ROOT = HOME + "/root"
-PATH = "%(ovs)s/utilities:%(ovs)s/ovsdb:%(ovs)s/vswitchd" % {"ovs": OVS_SRC}
-
-ENV["CFLAGS"] = "-g -O0"
-ENV["PATH"] = PATH + ":" + ENV["PATH"]
+if os.path.exists(PWD + "/WHY-OVS.md"):
+    OVS_SRC = PWD  # Use current directory as OVS source tree
+RUNDIR = OVS_SRC + "/_run"
+BUILD_GCC = OVS_SRC + "/_build-gcc"
+BUILD_CLANG = OVS_SRC + "/_build-clang"
 
 options = None
 parser = None
 commands = []
 
+def set_path(build):
+    PATH = "%(ovs)s/utilities:%(ovs)s/ovsdb:%(ovs)s/vswitchd" % {"ovs": build}
+
+    ENV["PATH"] = PATH + ":" + ENV["PATH"]
 
 def _sh(*args, **kwargs):
     print "------> " + " ".join(args)
@@ -50,15 +55,26 @@ def uname():
     return _sh("uname", "-r", capture=True)[0].strip()
 
 
+def sudo():
+    if os.geteuid() != 0:
+        _sh(" ".join(["sudo"] + sys.argv), check=True)
+        sys.exit(0)
+
 def conf():
     tag()
-    if options.clang:
-        ENV["CC"] = "clang"
 
-    configure = ["./configure", "--prefix=" + ROOT, "--localstatedir=" + ROOT,
-                 "--with-logdir=%s/log" % ROOT, "--with-rundir=%s/run" % ROOT,
-                 "--with-linux=/lib/modules/%s/build" % uname(),
-                 "--with-dbdir=" + ROOT]
+    try:
+        os.remove(OVS_SRC + "/Makefile")
+    except OSError:
+        pass
+
+    configure = ["../configure",
+                 "--prefix=" + RUNDIR, "--localstatedir=" + RUNDIR,
+                 "--with-logdir=%s/log" % RUNDIR,
+                 "--with-rundir=%s/run" % RUNDIR,
+                 "--enable-silent-rules", "--with-dbdir=" + RUNDIR, "--silent"]
+
+    cflags = "-g -fno-omit-frame-pointer"
 
     if options.werror:
         configure.append("--enable-Werror")
@@ -69,27 +85,89 @@ def conf():
     if options.mandir:
         configure.append("--mandir=" + options.mandir)
 
+    if options.with_dpdk:
+        configure.append("--with-dpdk=" + options.with_dpdk)
+        cflags += " -Wno-cast-align -Wno-bad-function-cast" # DPDK warnings.
+
+    if options.optimize is None:
+        options.optimize = 0
+
+    cflags += " -O%s" % str(options.optimize)
+
+    ENV["CFLAGS"] = cflags
+
     _sh("./boot.sh")
-    _sh(*configure)
+
+    try:
+        os.mkdir(BUILD_GCC)
+    except OSError:
+        pass # Directory exists.
+
+    os.chdir(BUILD_GCC)
+    _sh(*(configure + ["--with-linux=/lib/modules/%s/build" % uname()]))
+
+    try:
+        _sh("clang --version", check=True)
+        clang = True
+    except subprocess.CalledProcessError:
+        clang = False
+
+    try:
+        _sh("sparse --version", check=True)
+        sparse = True
+    except subprocess.CalledProcessError:
+        sparse = False
+
+    if clang:
+        try:
+            os.mkdir(BUILD_CLANG)
+        except OSError:
+            pass # Directory exists.
+
+        ENV["CC"] = "clang"
+        os.chdir(BUILD_CLANG)
+        _sh(*configure)
+
+    if sparse:
+        c1 = "C=1"
+    else:
+        c1 = ""
+
+    os.chdir(OVS_SRC)
+
+    make_str = "\t$(MAKE) -C %s $@\n"
+
+    mf = open(OVS_SRC + "/Makefile", "w")
+    mf.write("all:\n%:\n")
+    if clang:
+        mf.write(make_str % BUILD_CLANG)
+    mf.write("\t$(MAKE) -C %s %s $@\n" % (BUILD_GCC, c1))
+    mf.write("\ncheck-valgrind:\n")
+    mf.write("\ncheck:\n")
+    mf.write(make_str % BUILD_GCC)
+    mf.close()
 commands.append(conf)
 
 
 def make(args=""):
     make = "make -s -j 8 " + args
-    try:
-        _sh("cgcc", "--version", capture=True)
-        # XXX: For some reason the clang build doesn't place nicely with
-        # sparse.  At some point this needs to be figured out and this check
-        # removed.
-        if not options.clang:
-            make += " C=1"
-    except OSError:
-        pass
     _sh(make)
 commands.append(make)
 
 
 def check():
+    flags = ""
+    if options.jobs:
+        flags += "-j%d " % options.jobs
+    else:
+        flags += "-j8 "
+    if options.tests:
+        for arg in str.split(options.tests):
+            if arg[0].isdigit():
+                flags += "%s " % arg
+            else:
+                flags += "-k %s " % arg
+    ENV["TESTSUITEFLAGS"] = flags
     make("check")
 commands.append(check)
 
@@ -113,31 +191,34 @@ commands.append(tag)
 
 
 def kill():
+    sudo()
     for proc in ["ovs-vswitchd", "ovsdb-server"]:
-        if os.path.exists("%s/run/openvswitch/%s.pid" % (ROOT, proc)):
+        if os.path.exists("%s/run/openvswitch/%s.pid" % (RUNDIR, proc)):
             _sh("ovs-appctl", "-t", proc, "exit", check=False)
             time.sleep(.1)
-        _sh("sudo", "killall", "-q", "-2", proc, check=False)
+        _sh("killall", "-q", "-2", proc, check=False)
 commands.append(kill)
 
 
 def reset():
+    sudo()
     kill()
-    if os.path.exists(ROOT):
-        shutil.rmtree(ROOT)
+    if os.path.exists(RUNDIR):
+        shutil.rmtree(RUNDIR)
     for dp in _sh("ovs-dpctl dump-dps", capture=True):
         _sh("ovs-dpctl", "del-dp", dp.strip())
 commands.append(reset)
 
 
 def run():
+    sudo()
     kill()
     for d in ["log", "run"]:
-        d = "%s/%s" % (ROOT, d)
+        d = "%s/%s" % (RUNDIR, d)
         shutil.rmtree(d, ignore_errors=True)
         os.makedirs(d)
 
-    pki_dir = ROOT + "/pki"
+    pki_dir = RUNDIR + "/pki"
     if not os.path.exists(pki_dir):
         os.mkdir(pki_dir)
         os.chdir(pki_dir)
@@ -145,14 +226,25 @@ def run():
         _sh("ovs-pki req+sign ovsclient")
         os.chdir(OVS_SRC)
 
-    if not os.path.exists(ROOT + "/conf.db"):
-        _sh("ovsdb-tool", "create", ROOT + "/conf.db",
+    if not os.path.exists(RUNDIR + "/conf.db"):
+        _sh("ovsdb-tool", "create", RUNDIR + "/conf.db",
             OVS_SRC + "/vswitchd/vswitch.ovsschema")
 
-    opts = ["--pidfile", "--log-file", "--enable-dummy"]
+    opts = ["--pidfile", "--log-file"]
+
+    if (options.user == "") or (options.user == "root:root"):
+        _sh("chown", "root:root", "-R", RUNDIR)
+        if '--user' in sys.argv:
+           sys.argv.remove("--user")
+    else:
+        _sh("chown", options.user, "-R", RUNDIR);
+        opts = ["--user", options.user] + opts
+
+    if (options.monitor):
+        opts = ["--monitor"] + opts
 
     _sh(*(["ovsdb-server",
-           "--remote=punix:%s/run/db.sock" % ROOT,
+           "--remote=punix:%s/run/db.sock" % RUNDIR,
            "--remote=db:Open_vSwitch,Open_vSwitch,manager_options",
            "--private-key=db:Open_vSwitch,SSL,private_key",
            "--certificate=db:Open_vSwitch,SSL,certificate",
@@ -169,16 +261,22 @@ def run():
     _sh("ovs-vsctl --no-wait set Open_vSwitch %s ovs_version=%s"
         % (root_uuid, version))
 
-    cmd = [OVS_SRC + "/vswitchd/ovs-vswitchd"]
+    build = BUILD_CLANG if options.clang else BUILD_GCC
+    cmd = [build + "/vswitchd/ovs-vswitchd"]
+
+    if options.dpdk:
+        cmd.append("--dpdk")
+        cmd.extend(options.dpdk)
+        cmd.append("--")
+
     if options.gdb:
         cmd = ["gdb", "--args"] + cmd
     elif options.valgrind:
-        cmd = ["valgrind", "--track-origins=yes",
+        cmd = ["valgrind", "--track-origins=yes", "--leak-check=full",
                "--suppressions=%s/tests/glibc.supp" % OVS_SRC,
                "--suppressions=%s/tests/openssl.supp" % OVS_SRC] + cmd
     else:
-        cmd = ["sudo"] + cmd
-        opts = opts + ["-vconsole:off", "--detach"]
+        opts = opts + ["-vconsole:off", "--detach", "--enable-dummy"]
     _sh(*(cmd + opts))
 commands.append(run)
 
@@ -188,13 +286,15 @@ def modinst():
         print "Missing modules directory.  Is this a Linux system?"
         sys.exit(1)
 
+    sudo()
     try:
         _sh("rmmod", "openvswitch")
     except subprocess.CalledProcessError, e:
         pass  # Module isn't loaded
 
     try:
-        _sh("rm /lib/modules/%s/extra/openvswitch.ko" % uname())
+        _sh("rm -f /lib/modules/%s/extra/openvswitch.ko" % uname())
+        _sh("rm -f /lib/modules/%s/extra/vport-*.ko" % uname())
     except subprocess.CalledProcessError, e:
         pass  # Module isn't installed
 
@@ -204,6 +304,7 @@ def modinst():
 
     _sh("modprobe", "openvswitch")
     _sh("dmesg | grep openvswitch | tail -1")
+    _sh("find /lib/modules/%s/ -iname vport-*.ko -exec insmod '{}' \;" % uname())
 commands.append(modinst)
 
 
@@ -226,10 +327,10 @@ Basic Configuration:
 
     # First install the basic requirements needed to build Open vSwitch.
     sudo apt-get install git build-essential libtool autoconf pkg-config \\
-            libssl-dev pkg-config gdb linux-headers-`uname -r`
+            libssl-dev gdb libcap-ng-dev linux-headers-`uname -r`
 
     # Next clone the Open vSwitch source.
-    git clone git://git.openvswitch.org/openvswitch %(ovs)s
+    git clone https://github.com/openvswitch/ovs.git %(ovs)s
 
     # Setup environment variables.
     `%(v)s env`
@@ -240,6 +341,11 @@ Basic Configuration:
     # Install the kernel module
     sudo insmod %(ovs)s/datapath/linux/openvswitch.ko
 
+    # If needed, manually load all required vport modules:
+    sudo insmod %(ovs)s/datapath/linux/vport-vxlan.ko
+    sudo insmod %(ovs)s/datapath/linux/vport-geneve.ko
+    [...]
+
     # Run the switch.
     %(v)s run
 
@@ -254,10 +360,24 @@ Commands:
     modinst - Build ovs and install the kernel module.
     env     - Print the required path environment variable.
     doc     - Print this message.
-""" % {"ovs": OVS_SRC, "v": sys.argv[0], "run": ROOT}
+
+Note:
+    If running as non-root user, "kill", "reset", "run" and "modinst"
+    will always run as the root user, by rerun the commands with "sudo".
+""" % {"ovs": OVS_SRC, "v": sys.argv[0], "run": RUNDIR}
     sys.exit(0)
 commands.append(doc)
 
+def parse_subargs(option, opt_str, value, parser):
+    subopts = []
+
+    while parser.rargs:
+        dpdkarg = parser.rargs.pop(0)
+        if dpdkarg == "--":
+            break
+        subopts.append(dpdkarg)
+
+    setattr(parser.values, option.dest, subopts)
 
 def main():
     global options
@@ -277,6 +397,23 @@ def main():
                      action="store_true", help="configure with cached timing")
     group.add_option("--mandir", dest="mandir", metavar="MANDIR",
                      help="configure the man documentation install directory")
+    group.add_option("--with-dpdk", dest="with_dpdk", metavar="DPDK_BUILD",
+                     help="built with dpdk libraries located at DPDK_BUILD");
+    parser.add_option_group(group)
+
+    group = optparse.OptionGroup(parser, "Optimization Flags")
+    for i in ["s", "g"] + range(4) + ["fast"]:
+        group.add_option("--O%s" % str(i), dest="optimize",
+                         action="store_const", const=i,
+                         help="compile with -O%s" % str(i))
+    parser.add_option_group(group)
+
+    group = optparse.OptionGroup(parser, "check")
+    group.add_option("-j", "--jobs", dest="jobs", metavar="N", type="int",
+                     help="Run N tests in parallel")
+    group.add_option("--tests", dest="tests", metavar="FILTER",
+                     help="""run specific tests and/or a test category
+                          eg, --tests=\"1-10 megaflow\"""")
     parser.add_option_group(group)
 
     group = optparse.OptionGroup(parser, "run")
@@ -284,8 +421,17 @@ def main():
                      help="run ovs-vswitchd under gdb")
     group.add_option("--valgrind", dest="valgrind", action="store_true",
                      help="run ovs-vswitchd under valgrind")
+    group.add_option("--dpdk", dest="dpdk", action="callback",
+                     callback=parse_subargs,
+                     help="run ovs-vswitchd with dpdk subopts (ended by --)")
     group.add_option("--clang", dest="clang", action="store_true",
-                     help="build ovs-vswitchd with clang")
+                     help="Use binaries built by clang")
+    group.add_option("--user", dest="user", action="store", default="",
+                     help="run all daemons as a non root user")
+    group.add_option("--monitor", dest="monitor", action="store_true",
+                     help="run daemons with --monitor option")
+
+
     parser.add_option_group(group)
 
     options, args = parser.parse_args()
@@ -295,6 +441,11 @@ def main():
             print "Unknown argument " + arg
             doc()
 
+    if options.clang:
+        set_path(BUILD_CLANG)
+    else:
+        set_path(BUILD_GCC)
+
     try:
         os.chdir(OVS_SRC)
     except OSError: