tests+ci: Enable dual XML and HRF output of unit test results
This commit also makes unit tests to run against a debug build, to
ensure assertions are properly evaluated (based on suggestion
in I5f4419ea48d4eb333e9d107115ef3df4123f76e5).
Change-Id: I1bbf14f75aab155ed80a36fc4006f7a5d13f1289
Refs: #2252, #2805
diff --git a/tests/boost-multi-log-formatter.hpp b/tests/boost-multi-log-formatter.hpp
new file mode 100644
index 0000000..ae37416
--- /dev/null
+++ b/tests/boost-multi-log-formatter.hpp
@@ -0,0 +1,214 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2015 Regents of the University of California.
+ *
+ * Based on work by Martin Ba (http://stackoverflow.com/a/26718189)
+ *
+ * This file is distributed under the Boost Software License, Version 1.0.
+ * (See http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#ifndef NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
+#define NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
+
+#include <boost/version.hpp>
+
+#if BOOST_VERSION >= 105900
+#include <boost/test/unit_test_parameters.hpp>
+#else
+#include <boost/test/detail/unit_test_parameters.hpp>
+#endif // BOOST_VERSION >= 105900
+
+#include <boost/test/unit_test_log_formatter.hpp>
+#include <boost/test/output/compiler_log_formatter.hpp>
+#include <boost/test/output/xml_log_formatter.hpp>
+
+namespace boost {
+namespace unit_test {
+namespace output {
+
+/**
+ * @brief Log formatter for Boost.Test that outputs the logging to multiple formatters
+ *
+ * The log formatter is designed to output to one or multiple formatters at the same time. For
+ * example, one HRF formatter can output to the standard output, while XML formatter output to
+ * the file.
+ *
+ * Usage:
+ *
+ * // Call in init_unit_test_suite: (this will override the --log_format parameter)
+ * auto formatter = new boost::unit_test::output::multi_log_formatter; // same as already configured logger
+ *
+ * // Prepare and add additional logger(s)
+ * formatter.add(std::make_shared<boost::unit_test::output::xml_log_formatter>(),
+ * std::make_shared<std::ofstream>("out.xml"));
+ *
+ * boost::unit_test::unit_test_log.set_formatter(formatter);
+ *
+ * @note Calling `boost::unit_test::unit_test_log.set_stream(...)` will change the stream for
+ * the original logger.
+ */
+class multi_log_formatter : public unit_test_log_formatter
+{
+public:
+ /**
+ * @brief Create instance of the logger, based on the configured logger instance
+ */
+ multi_log_formatter()
+ {
+ auto format =
+#if BOOST_VERSION > 105900
+ runtime_config::get<output_format>(runtime_config::LOG_FORMAT);
+#else
+ runtime_config::log_format();
+#endif // BOOST_VERSION > 105900
+
+ switch (format) {
+ default:
+#if BOOST_VERSION >= 105900
+ case OF_CLF:
+#else
+ case CLF:
+#endif // BOOST_VERSION >= 105900
+ m_loggers.push_back({std::make_shared<compiler_log_formatter>(), nullptr});
+ break;
+#if BOOST_VERSION >= 105900
+ case OF_XML:
+#else
+ case XML:
+#endif // BOOST_VERSION >= 105900
+ m_loggers.push_back({std::make_shared<xml_log_formatter>(), nullptr});
+ break;
+ }
+ }
+
+ void
+ add(std::shared_ptr<unit_test_log_formatter> formatter, std::shared_ptr<std::ostream> os)
+ {
+ m_loggers.push_back({formatter, os});
+ }
+
+ // Formatter interface
+ void
+ log_start(std::ostream& os, counter_t test_cases_amount)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_start(l.os == nullptr ? os : *l.os, test_cases_amount);
+ }
+
+ void
+ log_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_finish(l.os == nullptr ? os : *l.os);
+ }
+
+ void
+ log_build_info(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_build_info(l.os == nullptr ? os : *l.os);
+ }
+
+ void
+ test_unit_start(std::ostream& os, const test_unit& tu)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_start(l.os == nullptr ? os : *l.os, tu);
+ }
+
+ void
+ test_unit_finish(std::ostream& os, const test_unit& tu, unsigned long elapsed)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_finish(l.os == nullptr ? os : *l.os, tu, elapsed);
+ }
+
+ void
+ test_unit_skipped(std::ostream& os, const test_unit& tu)
+ {
+ for (auto& l : m_loggers)
+ l.logger->test_unit_skipped(l.os == nullptr ? os : *l.os, tu);
+ }
+
+#if BOOST_VERSION >= 105900
+ void
+ log_exception_start(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception_start(l.os == nullptr ? os : *l.os, lcd, ex);
+ }
+
+ void
+ log_exception_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception_finish(l.os == nullptr ? os : *l.os);
+ }
+#else
+ void
+ log_exception(std::ostream& os, const log_checkpoint_data& lcd, const execution_exception& ex)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_exception(l.os == nullptr ? os : *l.os, lcd, ex);
+ }
+#endif // BOOST_VERSION >= 105900
+
+ void
+ log_entry_start(std::ostream& os, const log_entry_data& entry_data, log_entry_types let)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_start(l.os == nullptr ? os : *l.os, entry_data, let);
+ }
+
+ void
+ log_entry_value(std::ostream& os, const_string value)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_value(l.os == nullptr ? os : *l.os, value);
+ }
+
+ void
+ log_entry_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_finish(l.os == nullptr ? os : *l.os);
+ }
+
+#if BOOST_VERSION >= 105900
+ void
+ entry_context_start(std::ostream& os, log_level level)
+ {
+ for (auto& l : m_loggers)
+ l.logger->entry_context_start(l.os == nullptr ? os : *l.os, level);
+ }
+
+ void
+ log_entry_context(std::ostream& os, const_string value)
+ {
+ for (auto& l : m_loggers)
+ l.logger->log_entry_context(l.os == nullptr ? os : *l.os, value);
+ }
+
+ void
+ entry_context_finish(std::ostream& os)
+ {
+ for (auto& l : m_loggers)
+ l.logger->entry_context_finish(l.os == nullptr ? os : *l.os);
+ }
+#endif // BOOST_VERSION >= 105900
+
+private:
+ struct LoggerInfo
+ {
+ std::shared_ptr<unit_test_log_formatter> logger;
+ std::shared_ptr<std::ostream> os;
+ };
+ std::vector<LoggerInfo> m_loggers;
+};
+
+} // namespace output
+} // namespace unit_test
+} // namespace boost
+
+#endif // NDN_TESTS_BOOST_MULTI_LOG_FORMATTER_HPP
diff --git a/tests/main.cpp b/tests/main.cpp
index 2733634..49c3e9c 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -22,7 +22,84 @@
* NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
**/
-#define BOOST_TEST_MAIN 1
-#define BOOST_TEST_DYN_LINK 1
+#define BOOST_TEST_NO_MAIN
+#define BOOST_TEST_DYN_LINK
+#define BOOST_TEST_ALTERNATIVE_INIT_API
#include "boost-test.hpp"
+#include "boost-multi-log-formatter.hpp"
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/parsers.hpp>
+
+#include <fstream>
+#include <iostream>
+
+static bool
+init_tests()
+{
+ init_unit_test();
+
+ namespace po = boost::program_options;
+ namespace ut = boost::unit_test;
+
+ po::options_description extraOptions;
+ std::string logger;
+ std::string outputFile = "-";
+ extraOptions.add_options()
+ ("log_format2", po::value<std::string>(&logger), "Type of second log formatter: HRF or XML")
+ ("log_sink2", po::value<std::string>(&outputFile)->default_value(outputFile), "Second log sink, - for stdout")
+ ;
+ po::variables_map vm;
+ try {
+ po::store(po::command_line_parser(ut::framework::master_test_suite().argc,
+ ut::framework::master_test_suite().argv)
+ .options(extraOptions)
+ .run(),
+ vm);
+ po::notify(vm);
+ }
+ catch (const std::exception& e) {
+ std::cerr << "ERROR: " << e.what() << "\n"
+ << extraOptions << std::endl;
+ return false;
+ }
+
+ if (vm.count("log_format2") == 0) {
+ // second logger is not configured
+ return true;
+ }
+
+ std::shared_ptr<ut::unit_test_log_formatter> formatter;
+ if (logger == "XML") {
+ formatter = std::make_shared<ut::output::xml_log_formatter>();
+ }
+ else if (logger == "HRF") {
+ formatter = std::make_shared<ut::output::compiler_log_formatter>();
+ }
+ else {
+ std::cerr << "ERROR: only HRF or XML log formatter can be specified" << std::endl;
+ return false;
+ }
+
+ std::shared_ptr<std::ostream> output;
+ if (outputFile == "-") {
+ output = std::shared_ptr<std::ostream>(&std::cout, std::bind([]{}));
+ }
+ else {
+ output = std::make_shared<std::ofstream>(outputFile.c_str());
+ }
+
+ auto multiFormatter = new ut::output::multi_log_formatter;
+ multiFormatter->add(formatter, output);
+ ut::unit_test_log.set_formatter(multiFormatter);
+
+ return true;
+}
+
+int
+main(int argc, char* argv[])
+{
+ return ::boost::unit_test::unit_test_main(&init_tests, argc, argv);
+}
diff --git a/tests/other/wscript b/tests/other/wscript
index 51f8b6b..074e72c 100644
--- a/tests/other/wscript
+++ b/tests/other/wscript
@@ -27,8 +27,22 @@
top = '../..'
def build(bld):
- bld.program(target="../../cs-benchmark",
- source="cs-benchmark.cpp",
- use='daemon-objects unit-tests-main',
- install_path=None,
- )
+ for module, name in {"cs-benchmark": "CS Benchmark"}.iteritems():
+ # main()
+ bld(target='unit-tests-%s-main' % module,
+ name='unit-tests-%s-main' % module,
+ features='cxx',
+ use='BOOST',
+ source='../main.cpp',
+ defines=['BOOST_TEST_MODULE=%s' % name]
+ )
+
+ # unit-tests-%module
+ bld.program(
+ target='../../%s' % module,
+ features='cxx cxxprogram',
+ source=bld.path.ant_glob(['%s*.cpp' % module]),
+ use='daemon-objects unit-tests-base unit-tests-%s-main' % module,
+ includes='.',
+ install_path=None,
+ )
diff --git a/tests/wscript b/tests/wscript
index 7cfe4d2..823d25e 100644
--- a/tests/wscript
+++ b/tests/wscript
@@ -28,14 +28,6 @@
def build(bld):
# Unit tests
if bld.env['WITH_TESTS']:
- # main()
- unit_test_main = bld(
- target='unit-tests-main',
- name='unit-tests-main',
- features='cxx',
- use='core-objects',
- source='main.cpp',
- )
# common test modules
unit_test_base = bld(
@@ -47,46 +39,39 @@
headers='../common.hpp boost-test.hpp',
)
- # core tests
- unit_tests_core = bld.program(
- target='../unit-tests-core',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['core/**/*.cpp']),
- use='core-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ for module, name in {"core": "NFD Core Tests",
+ "daemon": "NFD Daemon Tests",
+ "rib": "NFD RIB Tests"}.iteritems():
+ # main()
+ bld(target='unit-tests-%s-main' % module,
+ name='unit-tests-%s-main' % module,
+ features='cxx',
+ use='BOOST',
+ source='main.cpp',
+ defines=['BOOST_TEST_MODULE=%s' % name]
+ )
- # NFD tests
- unit_tests_nfd = bld.program(
- target='../unit-tests-daemon',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['daemon/**/*.cpp'],
- excl=['daemon/face/ethernet*.cpp',
- 'daemon/face/unix*.cpp',
- 'daemon/face/websocket*.cpp']),
- use='daemon-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ # unit-tests-%module
+ unit_tests = bld.program(
+ target='../unit-tests-%s' % module,
+ features='cxx cxxprogram',
+ source=bld.path.ant_glob(['%s/**/*.cpp' % module],
+ excl=['%s/**/ethernet*.cpp' % module,
+ '%s/**/unix*.cpp' % module,
+ '%s/**/websocket*.cpp' % module]),
+ use='%s-objects unit-tests-base unit-tests-%s-main' % (module, module),
+ includes='.',
+ install_path=None,
+ )
- if bld.env['HAVE_LIBPCAP']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/ethernet*.cpp')
+ if bld.env['HAVE_LIBPCAP']:
+ unit_tests.source += bld.path.ant_glob('%s/**/ethernet*.cpp' % module)
- if bld.env['HAVE_UNIX_SOCKETS']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/unix*.cpp')
+ if bld.env['HAVE_UNIX_SOCKETS']:
+ unit_tests.source += bld.path.ant_glob('%s/**/unix*.cpp' % module)
- if bld.env['HAVE_WEBSOCKET']:
- unit_tests_nfd.source += bld.path.ant_glob('daemon/face/websocket*.cpp')
-
- unit_tests_rib = bld.program(
- target='../unit-tests-rib',
- features='cxx cxxprogram',
- source=bld.path.ant_glob(['rib/**/*.cpp']),
- use='rib-objects unit-tests-base unit-tests-main',
- includes='.',
- install_path=None,
- )
+ if bld.env['HAVE_WEBSOCKET']:
+ unit_tests.source += bld.path.ant_glob('%s/**/websocket*.cpp' % module)
# Other tests (e.g., stress tests that can be enabled even if unit tests are disabled)
if bld.env['WITH_TESTS'] or bld.env['WITH_OTHER_TESTS']: