tests+ci: Enable dual XML and HRF output of unit test results
This commit also makes unit tests to run against a debug build, to
ensure assertions are properly evaluated (based on suggestion
in I5f4419ea48d4eb333e9d107115ef3df4123f76e5).
Change-Id: I1bbf14f75aab155ed80a36fc4006f7a5d13f1289
Refs: #2252, #2805
diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh
index d22bfb6..2bc46fd 100755
--- a/.jenkins.d/01-ndn-cxx.sh
+++ b/.jenkins.d/01-ndn-cxx.sh
@@ -5,7 +5,7 @@
JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
source "$JDIR"/util.sh
-pushd /tmp >/dev/null
+pushd ${CACHE_DIR:-/tmp} >/dev/null
INSTALLED_VERSION=$((cd ndn-cxx && git rev-parse HEAD) 2>/dev/null || echo NONE)
diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh
index 1c4fdf2..09fa916 100755
--- a/.jenkins.d/10-build.sh
+++ b/.jenkins.d/10-build.sh
@@ -6,30 +6,31 @@
git submodule sync
git submodule update
-COVERAGE=$( python -c "print '--with-coverage' if 'code-coverage' in '$JOB_NAME' else ''" )
-
# Cleanup
sudo ./waf -j1 --color=yes distclean
-# Configure/build in debug mode
-./waf -j1 --color=yes configure --with-tests --debug
+# Configure/build in optimized mode with tests and precompiled headers
+./waf -j1 --color=yes configure --with-tests
./waf -j1 --color=yes build
# Cleanup
sudo ./waf -j1 --color=yes distclean
-# Configure/build in optimized mode without tests with precompiled headers
+# Configure/build in optimized mode without tests and with precompiled headers
./waf -j1 --color=yes configure
./waf -j1 --color=yes build
# Cleanup
sudo ./waf -j1 --color=yes distclean
-# Configure/build in optimized mode
-./waf -j1 --color=yes configure --with-tests --without-pch $COVERAGE
+# Configure/build in debug mode
+if [[ "$JOB_NAME" == *"code-coverage" ]]; then
+ COVERAGE="--with-coverage"
+fi
+./waf -j1 --color=yes configure --debug --with-tests --without-pch $COVERAGE
./waf -j1 --color=yes build
-# (tests will be run against optimized version)
+# (tests will be run against debug version)
# Install
sudo ./waf -j1 --color=yes install
diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh
index b4bf79a..dcd2b9d 100755
--- a/.jenkins.d/20-tests.sh
+++ b/.jenkins.d/20-tests.sh
@@ -2,19 +2,19 @@
set -x
set -e
+JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source "$JDIR"/util.sh
+
# Prepare environment
rm -Rf ~/.ndnx ~/.ndn
-echo $NODE_LABELS
-IS_OSX=$( python -c "print 'yes' if 'OSX' in '$NODE_LABELS'.strip().split(' ') else 'no'" )
-IS_LINUX=$( python -c "print 'yes' if 'Linux' in '$NODE_LABELS'.strip().split(' ') else 'no'" )
-
-if [[ $IS_OSX == "yes" ]]; then
+if has OSX $NODE_LABELS; then
security unlock-keychain -p "named-data"
sudo chgrp admin /dev/bpf*
sudo chmod g+rw /dev/bpf*
fi
-if [[ $IS_LINUX = "yes" ]]; then
+
+if has Linux $NODE_LABELS; then
sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-core || true
sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-daemon || true
sudo setcap cap_net_raw,cap_net_admin=eip `pwd`/build/unit-tests-rib || true
@@ -24,11 +24,18 @@
# Run unit tests
# Core
-./build/unit-tests-core -l test_suite
-sudo ./build/unit-tests-core -t TestPrivilegeHelper -l test_suite
+if [[ -n $XUNIT ]]; then
+ ./build/unit-tests-core -l all --log_format2=XML --log_sink2=build/xunit-core-report.xml
+ sudo ./build/unit-tests-core -t TestPrivilegeHelper -l all --log_format2=XML --log_sink2=build/xunit-core-sudo-report.xml
-# Daemon
-./build/unit-tests-daemon -l test_suite
+ ./build/unit-tests-daemon -l all --log_format2=XML --log_sink2=build/xunit-daemon-report.xml
-# RIB
-./build/unit-tests-rib -l test_suite
+ ./build/unit-tests-rib -l all --log_format2=XML --log_sink2=build/xunit-rib-report.xml
+else
+ ./build/unit-tests-core -l test_suite
+ sudo ./build/unit-tests-core -t TestPrivilegeHelper -l test_suite
+
+ ./build/unit-tests-daemon -l test_suite
+
+ ./build/unit-tests-rib -l test_suite
+fi
diff --git a/.jenkins.d/30-coverage.sh b/.jenkins.d/30-coverage.sh
index e462884..a00f8f1 100755
--- a/.jenkins.d/30-coverage.sh
+++ b/.jenkins.d/30-coverage.sh
@@ -2,9 +2,10 @@
set -x
set -e
-IS_COVR=$( python -c "print 'yes' if 'code-coverage' in '$JOB_NAME' else 'no'" )
+JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source "$JDIR"/util.sh
-if [[ $IS_COVR == "yes" ]]; then
+if [[ "$JOB_NAME" == *"code-coverage" ]]; then
BASE="`pwd | sed -e 's|/|\\\/|g'`\\"
(cd build && gcovr -x -f $BASE/core -f $BASE/daemon -f $BASE/rib -r ../ -o coverage.xml ./)
fi
diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md
index 956ae18..3bacba8 100644
--- a/.jenkins.d/README.md
+++ b/.jenkins.d/README.md
@@ -28,3 +28,6 @@
* empty: default build process
* `code-coverage` (Linux OS is assumed): build process with code coverage analysis
+
+- `CACHE_DIR`: the variable defines a path to folder containing cached files from previous builds,
+ e.g., a compiled version of ndn-cxx library. If not set, `/tmp` is used.
diff --git a/.waf-tools/coverage.py b/.waf-tools/coverage.py
index 0a3db65..ce92883 100644
--- a/.waf-tools/coverage.py
+++ b/.waf-tools/coverage.py
@@ -1,10 +1,6 @@
# -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
-#
-# Copyright (c) 2014, Regents of the University of California
-#
-# GPL 3.0 license, see the COPYING.md file for more information
-from waflib import TaskGen
+from waflib import TaskGen, Logs
def options(opt):
opt.add_option('--with-coverage', action='store_true', default=False, dest='with_coverage',
@@ -12,6 +8,8 @@
def configure(conf):
if conf.options.with_coverage:
+ if not conf.options.debug:
+ conf.fatal("Code coverage flags require debug mode compilation (add --debug)")
conf.check_cxx(cxxflags=['-fprofile-arcs', '-ftest-coverage', '-fPIC'],
linkflags=['-fprofile-arcs'], uselib_store='GCOV', mandatory=True)
diff --git a/tests/boost-multi-log-formatter.hpp b/tests/boost-multi-log-formatter.hpp
new file mode 100644
index 0000000..ae37416
--- /dev/null
+++ b/tests/boost-multi-log-formatter.hpp
@@ -0,0 +1,214 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2015 Regents of the University of California.
+ *
+ * Based on work by Martin Ba (http://stackoverflow.com/a/26718189)
+ *
+ * This file is distributed under the Boost Software License, Version 1.0.
+ * (See http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#ifndef NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
+#define NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
+
+#include <boost/version.hpp>
+
+#if BOOST_VERSION >= 105900
+#include <boost/test/unit_test_parameters.hpp>
+#else
+#include <boost/test/detail/unit_test_parameters.hpp>
+#endif // BOOST_VERSION >= 105900
+
+#include <boost/test/unit_test_log_formatter.hpp>
+#include <boost/test/output/compiler_log_formatter.hpp>
+#include <boost/test/output/xml_log_formatter.hpp>
+
+namespace boost {
+namespace unit_test {
+namespace output {
+
+/**
+ * @brief Log formatter for Boost.Test that outputs the logging to multiple formatters
+ *
+ * The log formatter is designed to output to one or multiple formatters at the same time. For
+ * example, one HRF formatter can output to the standard output, while XML formatter output to
+ * the file.
+ *
+ * Usage:
+ *
+ * // Call in init_unit_test_suite: (this will override the --log_format parameter)
+ * auto formatter = new boost::unit_test::output::multi_log_formatter; // same as already configured logger
+ *
+ * // Prepare and add additional logger(s)
+ * formatter.add(std::make_shared<boost::unit_test::output::xml_log_formatter>(),
+ * std::make_shared<std::ofstream>("out.xml"));
+ *
+ * boost::unit_test::unit_test_log.set_formatter(formatter);
+ *
+ * @note Calling `boost::unit_test::unit_test_log.set_stream(...)` will change the stream for
+ * the original logger.
+ */
+class multi_log_formatter : public unit_test_log_formatter
+{
+public:
+ /**
+ * @brief Create instance of the logger, based on the configured logger instance
+ */
+ multi_log_formatter()
+ {
+ auto format =
+#if BOOST_VERSION > 105900
+ runtime_config::get<output_format>(runtime_config::LOG_FORMAT);
+#else
+ runtime_config::log_format();
+#endif // BOOST_VERSION > 105900
+
+ switch (format) {
+ default:
+#if BOOST_VERSION >= 105900
+ case OF_CLF:
+#else
+ case CLF:
+#endif // BOOST_VERSION >= 105900
+ m_loggers.push_back({std::make_shared<compiler_log_formatter>(), nullptr});
+ break;
+#if BOOST_VERSION >= 105900
+ case OF_XML:
+#else
+ case XML:
+#endif // BOOST_VERSION >= 105900
+ m_loggers.push_back({std::make_shared<xml_log_formatter>(), nullptr});
+ break;
+ }
+ }
+
+ void
+ add(std::shared_ptr<unit_test_log_formatter> formatter, std::shared_ptr<std::ostream> os)
+ {
+ m_loggers.push_back({formatter, os});
+ }
+
+ // Formatter interface
+ void
+ log_start(std::ostream& os, counter_t test_cases_amount)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_start(l.os == nullptr ? os : *l.os, test_cases_amount);
+ }
+
+ void
+ log_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_finish(l.os == nullptr ? os : *l.os);
+ }
+
+ void
+ log_build_info(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_build_info(l.os == nullptr ? os : *l.os);
+ }
+
+ void
+ test_unit_start(std::ostream& os, const test_unit& tu)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_start(l.os == nullptr ? os : *l.os, tu);
+ }
+
+ void
+ test_unit_finish(std::ostream& os, const test_unit& tu, unsigned long elapsed)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_finish(l.os == nullptr ? os : *l.os, tu, elapsed);
+ }
+
+ void
+ test_unit_skipped(std::ostream& os, const test_unit& tu)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_skipped(l.os == nullptr ? os : *l.os, tu);
+ }
+
+#if BOOST_VERSION >= 105900
+ void
+ log_exception_start(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception_start(l.os == nullptr ? os : *l.os, lcd, ex);
+ }
+
+ void
+ log_exception_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception_finish(l.os == nullptr ? os : *l.os);
+ }
+#else
+ void
+ log_exception(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception(l.os == nullptr ? os : *l.os, lcd, ex);
+ }
+#endif // BOOST_VERSION >= 105900
+
+ void
+ log_entry_start(std::ostream& os, const log_entry_data& entry_data, log_entry_types let)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_start(l.os == nullptr ? os : *l.os, entry_data, let);
+ }
+
+ void
+ log_entry_value(std::ostream& os, const_string value)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_value(l.os == nullptr ? os : *l.os, value);
+ }
+
+ void
+ log_entry_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_finish(l.os == nullptr ? os : *l.os);
+ }
+
+#if BOOST_VERSION >= 105900
+ void
+ entry_context_start(std::ostream& os, log_level level)
+ {
+ for (auto& l : m_loggers)
+ l.logger->entry_context_start(l.os == nullptr ? os : *l.os, level);
+ }
+
+ void
+ log_entry_context(std::ostream& os, const_string value)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_context(l.os == nullptr ? os : *l.os, value);
+ }
+
+ void
+ entry_context_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->entry_context_finish(l.os == nullptr ? os : *l.os);
+ }
+#endif // BOOST_VERSION >= 105900
+
+private:
+ struct LoggerInfo
+ {
+ std::shared_ptr<unit_test_log_formatter> logger;
+ std::shared_ptr<std::ostream> os;
+ };
+ std::vector<LoggerInfo> m_loggers;
+};
+
+} // namespace output
+} // namespace unit_test
+} // namespace boost
+
+#endif // NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
diff --git a/tests/main.cpp b/tests/main.cpp
index 2733634..49c3e9c 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -22,7 +22,84 @@
* NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
**/
-#define BOOST_TEST_MAIN 1
-#define BOOST_TEST_DYN_LINK 1
+#define BOOST_TEST_NO_MAIN
+#define BOOST_TEST_DYN_LINK
+#define BOOST_TEST_ALTERNATIVE_INIT_API
#include "boost-test.hpp"
+#include "boost-multi-log-formatter.hpp"
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/parsers.hpp>
+
+#include <fstream>
+#include <iostream>
+
+static bool
+init_tests()
+{
+ init_unit_test();
+
+ namespace po = boost::program_options;
+ namespace ut = boost::unit_test;
+
+ po::options_description extraOptions;
+ std::string logger;
+ std::string outputFile = "-";
+ extraOptions.add_options()
+ ("log_format2", po::value<std::string>(&logger), "Type of second log formatter: HRF or XML")
+ ("log_sink2", po::value<std::string>(&outputFile)->default_value(outputFile), "Second log sink, - for stdout")
+ ;
+ po::variables_map vm;
+ try {
+ po::store(po::command_line_parser(ut::framework::master_test_suite().argc,
+ ut::framework::master_test_suite().argv)
+ .options(extraOptions)
+ .run(),
+ vm);
+ po::notify(vm);
+ }
+ catch (const std::exception& e) {
+ std::cerr << "ERROR: " << e.what() << "\n"
+ << extraOptions << std::endl;
+ return false;
+ }
+
+ if (vm.count("log_format2") == 0) {
+ // second logger is not configured
+ return true;
+ }
+
+ std::shared_ptr<ut::unit_test_log_formatter> formatter;
+ if (logger == "XML") {
+ formatter = std::make_shared<ut::output::xml_log_formatter>();
+ }
+ else if (logger == "HRF") {
+ formatter = std::make_shared<ut::output::compiler_log_formatter>();
+ }
+ else {
+ std::cerr << "ERROR: only HRF or XML log formatter can be specified" << std::endl;
+ return false;
+ }
+
+ std::shared_ptr<std::ostream> output;
+ if (outputFile == "-") {
+ output = std::shared_ptr<std::ostream>(&std::cout, std::bind([]{}));
+ }
+ else {
+ output = std::make_shared<std::ofstream>(outputFile.c_str());
+ }
+
+ auto multiFormatter = new ut::output::multi_log_formatter;
+ multiFormatter->add(formatter, output);
+ ut::unit_test_log.set_formatter(multiFormatter);
+
+ return true;
+}
+
+int
+main(int argc, char* argv[])
+{
+ return ::boost::unit_test::unit_test_main(&init_tests, argc, argv);
+}
diff --git a/tests/other/wscript b/tests/other/wscript
index 51f8b6b..074e72c 100644
--- a/tests/other/wscript
+++ b/tests/other/wscript
@@ -27,8 +27,22 @@
top = '../..'
def build(bld):
- bld.program(target="../../cs-benchmark",
- source="cs-benchmark.cpp",
- use='daemon-objects unit-tests-main',
- install_path=None,
- )
+ for module, name in {"cs-benchmark": "CS Benchmark"}.iteritems():
+ # main()
+ bld(target='unit-tests-%s-main' % module,
+ name='unit-tests-%s-main' % module,
+ features='cxx',
+ use='BOOST',
+ source='../main.cpp',
+ defines=['BOOST_TEST_MODULE=%s' % name]
+ )
+
+ # unit-tests-%module
+ bld.program(
+ target='../../%s' % module,
+ features='cxx cxxprogram',
+ source=bld.path.ant_glob(['%s*.cpp' % module]),
+ use='daemon-objects unit-tests-base unit-tests-%s-main' % module,
+ includes='.',
+ install_path=None,
+ )
diff --git a/tests/wscript b/tests/wscript
index 7cfe4d2..823d25e 100644
--- a/tests/wscript
+++ b/tests/wscript
@@ -28,14 +28,6 @@
def build(bld):
# Unit tests
if bld.env['WITH_TESTS']:
- # main()
- unit_test_main = bld(
- target='unit-tests-main',
- name='unit-tests-main',
- features='cxx',
- use='core-objects',
- source='main.cpp',
- )
# common test modules
unit_test_base = bld(
@@ -47,46 +39,39 @@
headers='../common.hpp boost-test.hpp',
)
- # core tests
- unit_tests_core = bld.program(
- target='../unit-tests-core',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['core/**/*.cpp']),
- use='core-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ for module, name in {"core": "NFD Core Tests",
+ "daemon": "NFD Daemon Tests",
+ "rib": "NFD RIB Tests"}.iteritems():
+ # main()
+ bld(target='unit-tests-%s-main' % module,
+ name='unit-tests-%s-main' % module,
+ features='cxx',
+ use='BOOST',
+ source='main.cpp',
+ defines=['BOOST_TEST_MODULE=%s' % name]
+ )
- # NFD tests
- unit_tests_nfd = bld.program(
- target='../unit-tests-daemon',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['daemon/**/*.cpp'],
- excl=['daemon/face/ethernet*.cpp',
- 'daemon/face/unix*.cpp',
- 'daemon/face/websocket*.cpp']),
- use='daemon-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ # unit-tests-%module
+ unit_tests = bld.program(
+ target='../unit-tests-%s' % module,
+ features='cxx cxxprogram',
+ source=bld.path.ant_glob(['%s/**/*.cpp' % module],
+ excl=['%s/**/ethernet*.cpp' % module,
+ '%s/**/unix*.cpp' % module,
+ '%s/**/websocket*.cpp' % module]),
+ use='%s-objects unit-tests-base unit-tests-%s-main' % (module, module),
+ includes='.',
+ install_path=None,
+ )
- if bld.env['HAVE_LIBPCAP']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/ethernet*.cpp')
+ if bld.env['HAVE_LIBPCAP']:
+ unit_tests.source += bld.path.ant_glob('%s/**/ethernet*.cpp' % module)
- if bld.env['HAVE_UNIX_SOCKETS']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/unix*.cpp')
+ if bld.env['HAVE_UNIX_SOCKETS']:
+ unit_tests.source += bld.path.ant_glob('%s/**/unix*.cpp' % module)
- if bld.env['HAVE_WEBSOCKET']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/websocket*.cpp')
-
- unit_tests_rib = bld.program(
- target='../unit-tests-rib',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['rib/**/*.cpp']),
- use='rib-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ if bld.env['HAVE_WEBSOCKET']:
+ unit_tests.source += bld.path.ant_glob('%s/**/websocket*.cpp' % module)
# Other tests (e.g., stress tests that can be enabled even if unit tests are disabled)
if bld.env['WITH_TESTS'] or bld.env['WITH_OTHER_TESTS']: