From 10b1980b6e734631d6e15a3e0411482e38923989 Mon Sep 17 00:00:00 2001 From: Lionel Sambuc Date: Thu, 27 Mar 2014 15:43:45 +0100 Subject: [PATCH] Add TAP output to kyua, see kyua help report-tap. Change-Id: Id64218b820da85b5dba75324c732f4e02b898849 --- distrib/sets/lists/minix/mi | 1 + .../bsd/kyua-cli/dist/cli/cmd_report_tap.cpp | 341 ++++++++++++++++++ .../bsd/kyua-cli/dist/cli/cmd_report_tap.hpp | 55 +++ external/bsd/kyua-cli/dist/cli/main.cpp | 2 + external/bsd/kyua-cli/lib/cli/Makefile | 1 + test/Makefile | 2 +- test/testkyua.sh | 21 ++ 7 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 external/bsd/kyua-cli/dist/cli/cmd_report_tap.cpp create mode 100644 external/bsd/kyua-cli/dist/cli/cmd_report_tap.hpp create mode 100755 test/testkyua.sh diff --git a/distrib/sets/lists/minix/mi b/distrib/sets/lists/minix/mi index 33a4204b8..b42e5ece4 100644 --- a/distrib/sets/lists/minix/mi +++ b/distrib/sets/lists/minix/mi @@ -5886,6 +5886,7 @@ ./usr/tests/minix-posix/test9 minix-sys ./usr/tests/minix-posix/testinterp minix-sys ./usr/tests/minix-posix/testisofs minix-sys +./usr/tests/minix-posix/testkyua minix-sys ./usr/tests/minix-posix/testmfs minix-sys ./usr/tests/minix-posix/testsh1 minix-sys ./usr/tests/minix-posix/testsh2 minix-sys diff --git a/external/bsd/kyua-cli/dist/cli/cmd_report_tap.cpp b/external/bsd/kyua-cli/dist/cli/cmd_report_tap.cpp new file mode 100644 index 000000000..8e24e176e --- /dev/null +++ b/external/bsd/kyua-cli/dist/cli/cmd_report_tap.cpp @@ -0,0 +1,341 @@ +// Copyright 2011 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cli/cmd_report.hpp" +#include "cli/cmd_report_tap.hpp" + +#include +#include +#include +#include +#include + +#include "cli/common.ipp" +#include "engine/action.hpp" +#include "engine/context.hpp" +#include "engine/drivers/scan_action.hpp" +#include "engine/test_result.hpp" +#include "utils/cmdline/exceptions.hpp" +#include "utils/cmdline/parser.ipp" +#include "utils/defs.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" + +namespace cmdline = utils::cmdline; +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scan_action = engine::drivers::scan_action; + +using cli::cmd_report_tap; +using utils::optional; + + +namespace { + + +/// Generates a plain-text report intended to be printed to the console. +class console_tap_hooks : public scan_action::base_hooks { + /// Indirection to print the output to the correct file stream. + cli::file_writer _writer; + + /// Whether to include the runtime context in the output or not. + const bool _show_context; + + /// Collection of result types to include in the report. + const cli::result_types& _results_filters; + + /// The action ID loaded. + int64_t _action_id; + + /// The total run time of the tests. + datetime::delta _runtime; + + /// Representation of a single result. + struct result_data { + /// The relative path to the test program. + fs::path binary_path; + + /// The name of the test case. + std::string test_case_name; + + /// The result of the test case. + engine::test_result result; + + /// The duration of the test case execution. + datetime::delta duration; + + /// Constructs a new results data. + /// + /// \param binary_path_ The relative path to the test program. + /// \param test_case_name_ The name of the test case. + /// \param result_ The result of the test case. + /// \param duration_ The duration of the test case execution. + result_data(const fs::path& binary_path_, + const std::string& test_case_name_, + const engine::test_result& result_, + const datetime::delta& duration_) : + binary_path(binary_path_), test_case_name(test_case_name_), + result(result_), duration(duration_) + { + } + }; + + /// Results received, broken down by their type. + /// + /// Note that this may not include all results, as keeping the whole list in + /// memory may be too much. + std::map< engine::test_result::result_type, + std::vector< result_data > > _results; + + /// Prints the execution context to the output. + /// + /// \param context The context to dump. + void + print_context(const engine::context& context) + { + _writer(" ===> Execution context"); + + _writer(F(" Current directory: %s") % context.cwd()); + const std::map< std::string, std::string >& env = context.env(); + if (env.empty()) + _writer(" No environment variables recorded"); + else { + _writer(" Environment variables:"); + for (std::map< std::string, std::string >::const_iterator + iter = env.begin(); iter != env.end(); iter++) { + _writer(F(" %s=%s") % (*iter).first % (*iter).second); + } + } + } + + /// Counts how many results of a given type have been received. + std::size_t + count_results(const engine::test_result::result_type type) + { + const std::map< engine::test_result::result_type, + std::vector< result_data > >::const_iterator iter = + _results.find(type); + if (iter == _results.end()) + return 0; + else + return (*iter).second.size(); + } + + /// Prints a set of results. + void + print_results(const engine::test_result::result_type type, + const char* title) + { + const std::map< engine::test_result::result_type, + std::vector< result_data > >::const_iterator iter2 = + _results.find(type); + if (iter2 == _results.end()) + return; + const std::vector< result_data >& all = (*iter2).second; + + std::string result = ""; + std::string comment = ""; + + if (type == engine::test_result::failed) + result = "not "; + + if (type == engine::test_result::broken) { + result = "not "; + comment = " # broken"; + } + + if (type == engine::test_result::skipped) + comment = " # skip"; + + _writer(F(" ===> %s") % title); + for (std::vector< result_data >::const_iterator iter = all.begin(); + iter != all.end(); iter++) { + _writer(F("%sok %s:%s%s") % + result % + (*iter).binary_path % + (*iter).test_case_name % + comment + ); + _writer(F(" %s:%s -> %s [%s]") % (*iter).binary_path % + (*iter).test_case_name % + cli::format_result((*iter).result) % + cli::format_delta((*iter).duration)); + } + } + +public: + /// Constructor for the hooks. + /// + /// \param ui_ The user interface object of the caller command. + /// \param outfile_ The file to which to send the output. + /// \param show_context_ Whether to include the runtime context in + /// the output or not. + /// \param results_filters_ The result types to include in the report. + /// Cannot be empty. + console_tap_hooks(cmdline::ui* ui_, const fs::path& outfile_, + const bool show_context_, + const cli::result_types& results_filters_) : + _writer(ui_, outfile_), + _show_context(show_context_), + _results_filters(results_filters_) + { + PRE(!results_filters_.empty()); + } + + /// Callback executed when an action is found. + /// + /// \param action_id The identifier of the loaded action. + /// \param action The action loaded from the database. + void + got_action(const int64_t action_id, const engine::action& action) + { + _action_id = action_id; + if (_show_context) + print_context(action.runtime_context()); + } + + /// Callback executed when a test results is found. + /// + /// \param iter Container for the test result's data. + void + got_result(store::results_iterator& iter) + { + _runtime += iter.duration(); + const engine::test_result result = iter.result(); + _results[result.type()].push_back( + result_data(iter.test_program()->relative_path(), + iter.test_case_name(), iter.result(), iter.duration())); + } + + /// Prints the tests summary. + void + print_tests(void) + { + using engine::test_result; + typedef std::map< test_result::result_type, const char* > types_map; + typedef std::map< test_result::result_type, std::size_t > types_counts; + + types_map titles; + titles[engine::test_result::broken] = "Broken tests"; + titles[engine::test_result::expected_failure] = "Expected failures"; + titles[engine::test_result::failed] = "Failed tests"; + titles[engine::test_result::passed] = "Passed tests"; + titles[engine::test_result::skipped] = "Skipped tests"; + + types_counts counts; + counts[engine::test_result::broken] = count_results(test_result::broken); + counts[engine::test_result::expected_failure] = count_results(test_result::expected_failure); + counts[engine::test_result::failed] = count_results(test_result::failed); + counts[engine::test_result::passed] = count_results(test_result::passed); + counts[engine::test_result::skipped] = count_results(test_result::skipped); + + const std::size_t total = + counts[engine::test_result::broken] + + counts[engine::test_result::expected_failure] + + counts[engine::test_result::failed] + + counts[engine::test_result::passed] + + counts[engine::test_result::skipped]; + + std::size_t selected_types_total = 0; + + for (cli::result_types::const_iterator iter = _results_filters.begin(); + iter != _results_filters.end(); ++iter) { + selected_types_total += counts.find(*iter)->second; + } + + _writer(F("1..%s") % selected_types_total); + for (cli::result_types::const_iterator iter = _results_filters.begin(); + iter != _results_filters.end(); ++iter) { + const types_map::const_iterator match = titles.find(*iter); + INV_MSG(match != titles.end(), "Conditional does not match user " + "input validation in parse_types()"); + print_results((*match).first, (*match).second); + } + + _writer(" ===> Summary"); + _writer(F(" Action: %s") % _action_id); + _writer(F(" Test cases: %s total, %s passed, %s skipped, %s expected failures, " + "%s broken, %s failed") % total % + counts[engine::test_result::passed] % + counts[engine::test_result::skipped] % + counts[engine::test_result::expected_failure] % + counts[engine::test_result::broken] % + counts[engine::test_result::failed]); + _writer(F(" Total time: %s") % cli::format_delta(_runtime)); + } +}; + + +} // anonymous namespace + + +/// Default constructor for cmd_report_tap. +cmd_report_tap::cmd_report_tap(void) : cli_command( + "report-tap", "", 0, 0, + "Generates a TAP report with the result of a " + "previous action") +{ + add_option(store_option); + add_option(cmdline::bool_option( + "show-context", "Include the execution context in the report")); + add_option(cmdline::int_option( + "action", "The action to report; if not specified, defaults to the " + "latest action in the database", "id")); + add_option(cmdline::path_option( + "output", "The file to which to write the report", + "path", "/dev/stdout")); + add_option(results_filter_option); +} + + +/// Entry point for the "report" subcommand. +/// +/// \param ui Object to interact with the I/O of the program. +/// \param cmdline Representation of the command line to the subcommand. +/// \param unused_user_config The runtime configuration of the program. +/// +/// \return 0 if everything is OK, 1 if the statement is invalid or if there is +/// any other problem. +int +cmd_report_tap::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline, + const config::tree& UTILS_UNUSED_PARAM(user_config)) +{ + optional< int64_t > action_id; + if (cmdline.has_option("action")) + action_id = cmdline.get_option< cmdline::int_option >("action"); + + const result_types types = get_result_types(cmdline); + console_tap_hooks hooks( + ui, cmdline.get_option< cmdline::path_option >("output"), + cmdline.has_option("show-context"), types); + scan_action::drive(store_path(cmdline), action_id, hooks); + hooks.print_tests(); + + return EXIT_SUCCESS; +} diff --git a/external/bsd/kyua-cli/dist/cli/cmd_report_tap.hpp b/external/bsd/kyua-cli/dist/cli/cmd_report_tap.hpp new file mode 100644 index 000000000..47d268fcd --- /dev/null +++ b/external/bsd/kyua-cli/dist/cli/cmd_report_tap.hpp @@ -0,0 +1,55 @@ +// Copyright 2012 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file cli/cmd_report_tap.hpp +/// Provides the cmd_report_tap class. + +#if !defined(CLI_CMD_REPORT_TAP_HPP) +#define CLI_CMD_REPORT_TAP_HPP + +#include "cli/common.hpp" +#include "utils/cmdline/ui.hpp" + +namespace cli { + + +/// Implementation of the "report-tap" subcommand. +class cmd_report_tap : public cli_command +{ +public: + cmd_report_tap(void); + + int run(utils::cmdline::ui*, const utils::cmdline::parsed_cmdline&, + const utils::config::tree&); +}; + + +} // namespace cli + + +#endif // !defined(CLI_CMD_REPORT_TAP_HPP) diff --git a/external/bsd/kyua-cli/dist/cli/main.cpp b/external/bsd/kyua-cli/dist/cli/main.cpp index 8263c1f6f..a32eeae78 100644 --- a/external/bsd/kyua-cli/dist/cli/main.cpp +++ b/external/bsd/kyua-cli/dist/cli/main.cpp @@ -51,6 +51,7 @@ extern "C" { #include "cli/cmd_list.hpp" #include "cli/cmd_report.hpp" #include "cli/cmd_report_html.hpp" +#include "cli/cmd_report_tap.hpp" #include "cli/cmd_test.hpp" #include "cli/common.ipp" #include "cli/config.hpp" @@ -169,6 +170,7 @@ safe_main(cmdline::ui* ui, int argc, const char* const argv[], commands.insert(new cli::cmd_report(), "Reporting"); commands.insert(new cli::cmd_report_html(), "Reporting"); + commands.insert(new cli::cmd_report_tap(), "Reporting"); if (mock_command.get() != NULL) commands.insert(mock_command); diff --git a/external/bsd/kyua-cli/lib/cli/Makefile b/external/bsd/kyua-cli/lib/cli/Makefile index 3f5ee1011..161ed2ea2 100644 --- a/external/bsd/kyua-cli/lib/cli/Makefile +++ b/external/bsd/kyua-cli/lib/cli/Makefile @@ -19,6 +19,7 @@ SRCS= cmd_about.cpp \ cmd_list.cpp \ cmd_report.cpp \ cmd_report_html.cpp \ + cmd_report_tap.cpp \ cmd_test.cpp \ common.cpp \ config.cpp \ diff --git a/test/Makefile b/test/Makefile index fc0c3c8a1..307941127 100644 --- a/test/Makefile +++ b/test/Makefile @@ -72,7 +72,7 @@ PROGS+= test${t} PROGS+= t10a t11a t11b t40a t40b t40c t40d t40e t40f t40g t60a t60b \ t67a t67b t68a t68b tvnd -SCRIPTS+= run testinterp.sh testsh1.sh testsh2.sh testmfs.sh testisofs.sh testvnd.sh +SCRIPTS+= run testinterp.sh testsh1.sh testsh2.sh testmfs.sh testisofs.sh testvnd.sh testkyua.sh # test57loop.S is not linked into the .bcl file. # This way, we can link it in when linking the final binary diff --git a/test/testkyua.sh b/test/testkyua.sh new file mode 100755 index 000000000..ebd7dc97f --- /dev/null +++ b/test/testkyua.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Execute and report the KYUA tests in TAP format + +# Execute only if kyua is available +if which kyua 2>&1 > /dev/null +then + cd /usr/tests + + kyua test 2>&1 > /dev/null + + # If some test fails, kyua return status is 1, which is why we cannot + # activate this before. + set -e + + # By default kyua omits successful tests. + kyua report-tap --results-filter=passed,skipped,xfail,broken,failed + + exit 0 +fi + +exit 1 -- 2.44.0