Merge branch 'automake-tests-parallel-v4' (*includes ci bits also*)

This patch set runs tests in parallel (on travis/appveyor/vagrant), it
includes cmake/autotools(automake).

It should significantly decrease time that tests tooks on travis-ci (3-4
times lower, right now it is about 14-17 hours - too long!), but not
without downsides, now because travis-ci workers has limited resources
we will have more timing-related failures, but this is another storry,
anyway ~16 hours is not acceptable. Anyway if machine has enough
resources it is great to have ability to run tests in parallel (which
automake couldn't do before).

Changes for common test env:
- autotools: before 17m / after 3m
- cmake:     before 15m / after 87 sec

* automake-tests-parallel-v4:
  Run tests in parallel (they are lightweight), on CI and vagrant boxes
  travis-ci: install cmake from xenial (CTEST_PARALLEL_LEVEL support)
  test: register different tests in automake
  test: run different tests under different options (in a wrapper)
  automake: do not use serial-tests if parallel-test-harness available

Fixes: libevent#439

Author: azat

.travis.yml

@@ -2,6 +2,7 @@ os:
   - linux
   - osx
 sudo: false
+dist: trusty
 
 env:
   matrix:
@@ -27,6 +28,8 @@ before_install:
   - if [ -n "$COVERALLS" ]; then
       pip install --user cpp-coveralls;
     fi
+  - export JOBS=20
+  - export TIMEOUT=50
   - if [ "$TRAVIS_OS_NAME" == "osx" ]; then
       brew update;
       brew uninstall libtool && brew install libtool;
@@ -43,32 +46,41 @@ before_install:
       export
         CFLAGS=-I$CMAKE_INCLUDE_PATH
         LDFLAGS=-L$CMAKE_LIBRARY_PATH;
+
+      export JOBS=4;
     fi
 
 addons:
   apt:
+    sources:
+    - xenial
+    - sourceline: 'deb http://archive.ubuntu.com/ubuntu xenial main'
     packages:
-      - zlib1g-dev
-      - libssl-dev
-      - build-essential
-      - automake
-      - autoconf
-      - cmake
-      - lcov
+    - zlib1g-dev
+    - libssl-dev
+    - build-essential
+    - automake
+    - autoconf
+    - cmake
+    - lcov
 
 script:
   - if [ "$EVENT_BUILD_METHOD" = "autotools" ]; then
       ./autogen.sh &&
       ./configure $EVENT_CONFIGURE_OPTIONS &&
       make &&
-      make verify;
+      travis_wait $TIMEOUT make -j $JOBS verify;
     fi
   - if [ "$EVENT_BUILD_METHOD" = "cmake" ]; then
+      export
+        CTEST_PARALLEL_LEVEL=$JOBS
+        CTEST_OUTPUT_ON_FAILURE=1;
+
       mkdir build &&
       cd build &&
       cmake .. $EVENT_CMAKE_OPTIONS &&
-      cmake --build . &&
-      CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target verify;
+      travis_wait $TIMEOUT
+      cmake --build . --target verify;
     fi
 
 after_script:

Makefile.am

@@ -4,6 +4,10 @@
 #
 # See LICENSE for copying information.
 
+# 'foreign' means that we're not enforcing GNU package rules strictly.
+# '1.9' means that we need automake 1.9 or later (and we do).
+AUTOMAKE_OPTIONS = foreign 1.9 subdir-objects
+
 ACLOCAL_AMFLAGS = -I m4
 
 # This is the "Release" of the Libevent ABI.  It takes precedence over

Vagrantfile

@@ -61,7 +61,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -71,7 +71,7 @@ Vagrant.configure("2") do |config|
         cd /vagrant
         ./autogen.sh
         ./configure
-        make -j4 verify
+        make -j20 verify
       SHELL
     end
   end
@@ -102,7 +102,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -112,7 +112,7 @@ Vagrant.configure("2") do |config|
         cd /vagrant
         ./autogen.sh
         ./configure
-        make -j4 verify
+        make -j20 verify
       SHELL
     end
   end
@@ -141,7 +141,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -151,7 +151,7 @@ Vagrant.configure("2") do |config|
         cd /vagrant
         ./autogen.sh
         ./configure
-        make -j4 verify
+        make -j20 verify
       SHELL
     end
   end
@@ -200,7 +200,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -212,7 +212,7 @@ Vagrant.configure("2") do |config|
         cd ~/vagrant
         ./autogen.sh
         MAKE=gmake ./configure
-        gmake -j4 verify
+        gmake -j20 verify
       SHELL
     end
   end
@@ -249,7 +249,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -263,7 +263,7 @@ Vagrant.configure("2") do |config|
         cd /vagrant
         ./autogen.sh
         ./configure
-        make -j4 verify
+        make -j20 verify
       SHELL
     end
   end
@@ -300,7 +300,7 @@ Vagrant.configure("2") do |config|
 
         export CTEST_TEST_TIMEOUT=1800
         export CTEST_OUTPUT_ON_FAILURE=1
-        export CTEST_PARALLEL_LEVEL=10
+        export CTEST_PARALLEL_LEVEL=20
         cmake --build . --target verify
       SHELL
     end
@@ -310,7 +310,7 @@ Vagrant.configure("2") do |config|
         cd /vagrant
         ./autogen.sh
         ./configure
-        make -j4 verify
+        make -j20 verify
       SHELL
     end
   end
@@ -390,7 +390,7 @@ Vagrant.configure("2") do |config|
         bash -lc "echo 'C:/OpenSSL-Win32 /ssl ntfs binary 0 0' >> /etc/fstab"
         bash -lc "echo 'C:/vagrant /vagrant ntfs binary 0 0' >> /etc/fstab"
 
-        bash -lc "exec 0</dev/null; exec 2>&1; cd /vagrant; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include && make -j4 verify"
+        bash -lc "exec 0</dev/null; exec 2>&1; cd /vagrant; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include && make -j20 verify"
       SHELL
     end
   end

appveyor.yml

@@ -49,11 +49,12 @@ build_script:
         bash -lc "echo 'C:\MinGW /mingw' > /etc/fstab"
         bash -lc "echo 'C:\OpenSSL-Win32 /ssl' >> /etc/fstab"
         $env:APPVEYOR_BUILD_FOLDER = $env:APPVEYOR_BUILD_FOLDER -replace "\\", "/"
-        bash -lc "exec 0</dev/null; exec 2>&1; cd $env:APPVEYOR_BUILD_FOLDER; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include $env:EVENT_CONFIGURE_OPTIONS && make && make verify"
+        bash -lc "exec 0</dev/null; exec 2>&1; cd $env:APPVEYOR_BUILD_FOLDER; bash -x ./autogen.sh && ./configure LDFLAGS='-L/ssl -L/ssl/lib -L/ssl/lib/MinGW' CFLAGS=-I/ssl/include $env:EVENT_CONFIGURE_OPTIONS && make && make verify -j20"
       } else {
         md build
         cd build
         cmake .. $env:EVENT_CMAKE_OPTIONS
         cmake --build .
+        $env:CTEST_PARALLEL_LEVEL="20"
         ctest --output-on-failure
       }

configure.ac

@@ -10,18 +10,7 @@ AC_PREREQ(2.59)
 AC_CONFIG_SRCDIR(event.c)
 
 AC_CONFIG_MACRO_DIR([m4])
-
-# 'foreign' means that we're not enforcing GNU package rules strictly.
-# '1.9' means that we need automake 1.9 or later (and we do).
-# serial-tests means that we don't need parallel test harness
-AM_INIT_AUTOMAKE(m4_esyscmd([echo foreign 1.9 subdir-objects
-    case `automake --version | head -n 1` in
-    *1.9*);;
-    *1.10*);;
-    *1.11*);;
-    *) echo serial-tests;;
-    esac]))
-
+AM_INIT_AUTOMAKE
 dnl AM_SILENT_RULES req. automake 1.11.  [no] defaults V=1
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_CONFIG_HEADERS(config.h  evconfig-private.h:evconfig-private.h.in)

test/include.am

@@ -42,14 +42,41 @@ noinst_HEADERS+=				\
 	test/tinytest_local.h			\
 	test/tinytest_macros.h
 
-# We need to copy this file, since automake doesn't want us to use top_srcdir
-# in TESTS.
-TESTS = test/test-script.sh
+TESTS = \
+	test_runner_epoll \
+	test_runner_select \
+	test_runner_kqueue \
+	test_runner_evport \
+	test_runner_devpoll \
+	test_runner_poll \
+	test_runner_win32 \
+	test_runner_timerfd \
+	test_runner_changelist \
+	test_runner_timerfd_changelist
+LOG_COMPILER = true
+TESTS_COMPILER = true
+
+test_runner_epoll: test/test.sh
+	test/test.sh -b EPOLL
+test_runner_select: test/test.sh
+	test/test.sh -b SELECT
+test_runner_kqueue: test/test.sh
+	test/test.sh -b KQUEUE
+test_runner_evport: test/test.sh
+	test/test.sh -b EVPORT
+test_runner_devpoll: test/test.sh
+	test/test.sh -b DEVPOLL
+test_runner_poll: test/test.sh
+	test/test.sh -b POLL
+test_runner_win32: test/test.sh
+	test/test.sh -b WIN32
+test_runner_timerfd: test/test.sh
+	test/test.sh -b "" -t
+test_runner_changelist: test/test.sh
+	test/test.sh -b "" -c
+test_runner_timerfd_changelist: test/test.sh
+	test/test.sh -b "" -T
 
-test/test-script.sh: test/test.sh
-	cp $(top_srcdir)/test/test.sh $@
-
-DISTCLEANFILES += test/test-script.sh
 DISTCLEANFILES += test/regress.gen.c test/regress.gen.h
 
 if BUILD_REGRESS

test/test.sh

@@ -28,7 +28,7 @@ fi
 TEST_DIR=.
 TEST_SRC_DIR=.
 
-T=`echo "$0" | sed -e 's/test.sh$//' | sed -e 's/test-script.sh//' `
+T=`echo "$0" | sed -e 's/test.sh$//'`
 if test -x "$T/test-init"
 then
 	TEST_DIR="$T"
@@ -146,15 +146,43 @@ do_test() {
 	run_tests
 }
 
-announce "Running tests:"
+usage()
+{
+	cat <<EOL
+  -b   - specify backends
+  -t   - run timerfd test
+  -c   - run changelist test
+  -T   - run timerfd+changelist test
+EOL
+}
+main()
+{
+	backends=$BACKENDS
+	timerfd=0
+	changelist=0
+	timerfd_changelist=0
+
+	while getopts "b:tcT" c; do
+		case "$c" in
+			b) backends="$OPTARG";;
+			t) timerfd=1;;
+			c) changelist=1;;
+			T) timerfd_changelist=1;;
+			?*) usage && exit 1;;
+		esac
+	done
 
-do_test EPOLL "(timerfd)"
-do_test EPOLL "(changelist)"
-do_test EPOLL "(timerfd+changelist)"
-for i in $BACKENDS; do
-	do_test $i
-done
+	announce "Running tests:"
 
-if test "$FAILED" = "yes"; then
-	exit 1
-fi
+	[ $timerfd -eq 0 ] || do_test EPOLL "(timerfd)"
+	[ $changelist -eq 0 ] || do_test EPOLL "(changelist)"
+	[ $timerfd_changelist -eq 0 ] || do_test EPOLL "(timerfd+changelist)"
+	for i in $backends; do
+		do_test $i
+	done
+
+	if test "$FAILED" = "yes"; then
+		exit 1
+	fi
+}
+main "$@"