tests: Add ability to automatically rerun failed tests.

[cascardo/ovs.git] / tests / automake.mk

diff --git a/tests/automake.mk b/tests/automake.mk
index c2ef9a4..592f648 100644 (file)
--- a/tests/automake.mk
+++ b/tests/automake.mk
@@ -1,9 +1,12 @@
 EXTRA_DIST += \
        $(COMMON_MACROS_AT) \
        $(TESTSUITE_AT) \
-       $(KMOD_TESTSUITE_AT) \
+       $(SYSTEM_TESTSUITE_AT) \
+       $(SYSTEM_KMOD_TESTSUITE_AT) \
+       $(SYSTEM_USERSPACE_TESTSUITE_AT) \
        $(TESTSUITE) \
-       $(KMOD_TESTSUITE) \
+       $(SYSTEM_KMOD_TESTSUITE) \
+       $(SYSTEM_USERSPACE_TESTSUITE) \
        tests/atlocal.in \
        $(srcdir)/package.m4 \
        $(srcdir)/tests/testsuite \
@@ -47,6 +50,7 @@ TESTSUITE_AT = \
        tests/jsonrpc-py.at \
        tests/tunnel.at \
        tests/tunnel-push-pop.at \
+       tests/tunnel-push-pop-ipv6.at \
        tests/lockfile.at \
        tests/reconnect.at \
        tests/ovs-vswitchd.at \
@@ -83,22 +87,36 @@ TESTSUITE_AT = \
        tests/vlog.at \
        tests/vtep-ctl.at \
        tests/auto-attach.at \
-       tests/ovn.at
+       tests/ovn.at \
+       tests/ovn-nbctl.at \
+       tests/ovn-sbctl.at \
+       tests/ovn-controller.at \
+       tests/ovn-controller-vtep.at
 
-KMOD_TESTSUITE_AT = \
-       tests/kmod-testsuite.at \
-       tests/kmod-macros.at \
-       tests/traffic.at
+SYSTEM_KMOD_TESTSUITE_AT = \
+       tests/system-common-macros.at \
+       tests/system-kmod-testsuite.at \
+       tests/system-kmod-macros.at
+
+SYSTEM_USERSPACE_TESTSUITE_AT = \
+       tests/system-userspace-testsuite.at \
+       tests/system-userspace-macros.at
+
+SYSTEM_TESTSUITE_AT = \
+       tests/system-common-macros.at \
+       tests/system-traffic.at
 
 TESTSUITE = $(srcdir)/tests/testsuite
 TESTSUITE_PATCH = $(srcdir)/tests/testsuite.patch
-KMOD_TESTSUITE = $(srcdir)/tests/kmod-testsuite
+SYSTEM_KMOD_TESTSUITE = $(srcdir)/tests/system-kmod-testsuite
+SYSTEM_USERSPACE_TESTSUITE = $(srcdir)/tests/system-userspace-testsuite
 DISTCLEANFILES += tests/atconfig tests/atlocal
 
-AUTOTEST_PATH = utilities:vswitchd:ovsdb:vtep:tests:$(PTHREAD_WIN32_DIR_DLL)
+AUTOTEST_PATH = utilities:vswitchd:ovsdb:vtep:tests:$(PTHREAD_WIN32_DIR_DLL):ovn/controller-vtep:ovn/northd:ovn/utilities:ovn/controller
 
 check-local: tests/atconfig tests/atlocal $(TESTSUITE)
-       $(SHELL) '$(TESTSUITE)' -C tests AUTOTEST_PATH=$(AUTOTEST_PATH) $(TESTSUITEFLAGS)
+       set $(SHELL) '$(TESTSUITE)' -C tests AUTOTEST_PATH=$(AUTOTEST_PATH) $(TESTSUITEFLAGS); \
+       "$$@" || (test X'$(RECHECK)' = Xyes && "$$@" --recheck)
 \f
 # Python Coverage support.
 # Requires coverage.py http://nedbatchelder.com/code/coverage/.
@@ -153,6 +171,7 @@ valgrind_wrappers = \
        tests/valgrind/test-stp \
        tests/valgrind/test-type-props \
        tests/valgrind/test-unix-socket \
+       tests/valgrind/test-unixctl \
        tests/valgrind/test-uuid \
        tests/valgrind/test-vconn
 
@@ -189,15 +208,18 @@ check-ryu: all
 EXTRA_DIST += tests/run-ryu
 \f
 # Run kmod tests. Assume kernel modules has been installed or linked into the kernel
-check-kernel: all tests/atconfig tests/atlocal $(KMOD_TESTSUITE)
-       $(SHELL) '$(KMOD_TESTSUITE)' -C tests  AUTOTEST_PATH='$(AUTOTEST_PATH)' -d $(TESTSUITEFLAGS)
+check-kernel: all tests/atconfig tests/atlocal $(SYSTEM_KMOD_TESTSUITE)
+       $(SHELL) '$(SYSTEM_KMOD_TESTSUITE)' -C tests  AUTOTEST_PATH='$(AUTOTEST_PATH)' -d $(TESTSUITEFLAGS)
 
 # Testing the out of tree Kernel module
-check-kmod: all tests/atconfig tests/atlocal $(KMOD_TESTSUITE)
+check-kmod: all tests/atconfig tests/atlocal $(SYSTEM_KMOD_TESTSUITE)
        $(MAKE) modules_install
        modprobe -r openvswitch
        $(MAKE) check-kernel
 
+check-system-userspace: all tests/atconfig tests/atlocal $(SYSTEM_USERSPACE_TESTSUITE)
+       $(SHELL) '$(SYSTEM_USERSPACE_TESTSUITE)' -C tests  AUTOTEST_PATH='$(AUTOTEST_PATH)' $(TESTSUITEFLAGS)
+
 clean-local:
        test ! -f '$(TESTSUITE)' || $(SHELL) '$(TESTSUITE)' -C tests --clean
 
@@ -214,7 +236,11 @@ $(TESTSUITE): package.m4 $(TESTSUITE_AT) $(COMMON_MACROS_AT)
        $(AM_V_at)mv $@.tmp $@
 endif
 
-$(KMOD_TESTSUITE): package.m4 $(KMOD_TESTSUITE_AT) $(COMMON_MACROS_AT)
+$(SYSTEM_KMOD_TESTSUITE): package.m4 $(SYSTEM_TESTSUITE_AT) $(SYSTEM_KMOD_TESTSUITE_AT) $(COMMON_MACROS_AT)
+       $(AM_V_GEN)$(AUTOTEST) -I '$(srcdir)' -o $@.tmp $@.at
+       $(AM_V_at)mv $@.tmp $@
+
+$(SYSTEM_USERSPACE_TESTSUITE): package.m4 $(SYSTEM_TESTSUITE_AT) $(SYSTEM_USERSPACE_TESTSUITE_AT) $(COMMON_MACROS_AT)
        $(AM_V_GEN)$(AUTOTEST) -I '$(srcdir)' -o $@.tmp $@.at
        $(AM_V_at)mv $@.tmp $@
 
@@ -289,6 +315,7 @@ tests_ovstest_SOURCES = \
        tests/test-sflow.c \
        tests/test-sha1.c \
        tests/test-stp.c \
+       tests/test-unixctl.c \
        tests/test-util.c \
        tests/test-uuid.c \
        tests/test-bitmap.c \
@@ -300,6 +327,11 @@ tests_ovstest_SOURCES += \
        tests/test-unix-socket.c
 endif
 
+if LINUX
+tests_ovstest_SOURCES += \
+       tests/test-netlink-conntrack.c
+endif
+
 tests_ovstest_LDADD = lib/libopenvswitch.la ovn/lib/libovn.la
 dist_check_SCRIPTS = tests/flowgen.pl
 
@@ -315,6 +347,7 @@ CHECK_PYFILES = \
        tests/test-daemon.py \
        tests/test-json.py \
        tests/test-jsonrpc.py \
+       tests/test-l7.py \
        tests/test-ovsdb.py \
        tests/test-reconnect.py \
        tests/MockXenAPI.py \
@@ -324,6 +357,8 @@ CHECK_PYFILES = \
 EXTRA_DIST += $(CHECK_PYFILES)
 PYCOV_CLEAN_FILES += $(CHECK_PYFILES:.py=.py,cover) .coverage
 
+FLAKE8_PYFILES += $(CHECK_PYFILES)
+
 if HAVE_OPENSSL
 TESTPKI_FILES = \
        tests/testpki-cacert.pem \