LCOV - code coverage report
Current view: top level - test - fixture.cpp (source / functions) Hit Total Coverage
Test: lcov.info Lines: 154 252 61.1 %
Date: 2024-04-28 12:00:40 Functions: 28 36 77.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*
       2             :  * Cppcheck - A tool for static C/C++ code analysis
       3             :  * Copyright (C) 2007-2024 Cppcheck team.
       4             :  *
       5             :  * This program is free software: you can redistribute it and/or modify
       6             :  * it under the terms of the GNU General Public License as published by
       7             :  * the Free Software Foundation, either version 3 of the License, or
       8             :  * (at your option) any later version.
       9             :  *
      10             :  * This program is distributed in the hope that it will be useful,
      11             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      12             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      13             :  * GNU General Public License for more details.
      14             :  *
      15             :  * You should have received a copy of the GNU General Public License
      16             :  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
      17             :  */
      18             : 
      19             : #include "fixture.h"
      20             : 
      21             : #include "cppcheck.h"
      22             : #include "errortypes.h"
      23             : #include "options.h"
      24             : #include "redirect.h"
      25             : 
      26             : #include <algorithm>
      27             : #include <cstdio>
      28             : #include <cctype>
      29             : #include <exception>
      30             : #include <iostream>
      31             : #include <set>
      32             : #include <sstream>
      33             : #include <string>
      34             : 
      35             : #include "xml.h"
      36             : 
      37             : /**
      38             :  * TestRegistry
      39             :  **/
      40             : namespace {
      41             :     struct CompareFixtures {
      42         714 :         bool operator()(const TestFixture* lhs, const TestFixture* rhs) const {
      43         714 :             return lhs->classname < rhs->classname;
      44             :         }
      45             :     };
      46             : }
      47             : using TestSet = std::set<TestFixture*, CompareFixtures>;
      48             : namespace {
      49             :     class TestRegistry {
      50             :         TestSet _tests;
      51             :     public:
      52             : 
      53          78 :         static TestRegistry &theInstance() {
      54          78 :             static TestRegistry testreg;
      55          78 :             return testreg;
      56             :         }
      57             : 
      58          77 :         void addTest(TestFixture *t) {
      59          77 :             _tests.insert(t);
      60          77 :         }
      61             : 
      62           1 :         const TestSet &tests() const {
      63           1 :             return _tests;
      64             :         }
      65             :     };
      66             : }
      67             : 
      68             : 
      69             : 
      70             : 
      71             : /**
      72             :  * TestFixture
      73             :  **/
      74             : 
      75             : std::ostringstream TestFixture::errmsg;
      76             : unsigned int TestFixture::countTests;
      77             : 
      78             : std::size_t TestFixture::fails_counter = 0;
      79             : std::size_t TestFixture::todos_counter = 0;
      80             : std::size_t TestFixture::succeeded_todos_counter = 0;
      81             : 
      82          77 : TestFixture::TestFixture(const char * const _name)
      83          77 :     : classname(_name)
      84             : {
      85          77 :     TestRegistry::theInstance().addTest(this);
      86          77 : }
      87             : 
      88             : 
      89        4701 : bool TestFixture::prepareTest(const char testname[])
      90             : {
      91        4701 :     mVerbose = false;
      92        4701 :     mTemplateFormat.clear();
      93        4701 :     mTemplateLocation.clear();
      94        4701 :     CppCheck::resetTimerResults();
      95             : 
      96        4701 :     prepareTestInternal();
      97             : 
      98             :     // Check if tests should be executed
      99        4701 :     if (testToRun.empty() || testToRun == testname) {
     100             :         // Tests will be executed - prepare them
     101        4701 :         mTestname = testname;
     102        4701 :         ++countTests;
     103        4701 :         if (quiet_tests) {
     104           0 :             std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
     105           0 :             std::fflush(stdout);
     106             :         } else {
     107        4701 :             std::cout << classname << "::" << mTestname << std::endl;
     108             :         }
     109        4701 :         return !dry_run;
     110             :     }
     111           0 :     return false;
     112             : }
     113             : 
     114        4701 : void TestFixture::teardownTest()
     115             : {
     116        4701 :     teardownTestInternal();
     117             : 
     118             :     {
     119        9402 :         const std::string s = errout_str();
     120        4701 :         if (!s.empty())
     121           0 :             throw std::runtime_error("unconsumed ErrorLogger err: " + s);
     122             :     }
     123             :     {
     124        9402 :         const std::string s = output_str();
     125        4701 :         if (!s.empty())
     126           0 :             throw std::runtime_error("unconsumed ErrorLogger out: " + s);
     127             :     }
     128        4701 : }
     129             : 
     130           0 : std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
     131             : {
     132           0 :     return std::string(filename) + ':' + std::to_string(linenr) + '(' + classname + "::" + mTestname + ')';
     133             : }
     134             : 
     135           0 : static std::string writestr(const std::string &str, bool gccStyle = false)
     136             : {
     137           0 :     std::ostringstream ostr;
     138           0 :     if (gccStyle)
     139           0 :         ostr << '\"';
     140           0 :     for (std::string::const_iterator i = str.cbegin(); i != str.cend(); ++i) {
     141           0 :         if (*i == '\n') {
     142           0 :             ostr << "\\n";
     143           0 :             if ((i+1) != str.end() && !gccStyle)
     144           0 :                 ostr << std::endl;
     145           0 :         } else if (*i == '\t')
     146           0 :             ostr << "\\t";
     147           0 :         else if (*i == '\"')
     148           0 :             ostr << "\\\"";
     149           0 :         else if (std::isprint(static_cast<unsigned char>(*i)))
     150           0 :             ostr << *i;
     151             :         else
     152           0 :             ostr << "\\x" << std::hex << short{*i};
     153             :     }
     154           0 :     if (!str.empty() && !gccStyle)
     155           0 :         ostr << std::endl;
     156           0 :     else if (gccStyle)
     157           0 :         ostr << '\"';
     158           0 :     return ostr.str();
     159             : }
     160             : 
     161       22634 : bool TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const
     162             : {
     163       22634 :     if (!condition) {
     164           0 :         ++fails_counter;
     165           0 :         errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
     166             :     }
     167       22634 :     return condition;
     168             : }
     169             : 
     170           0 : void TestFixture::assertEqualsFailed(const char* const filename, const unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const
     171             : {
     172           0 :     ++fails_counter;
     173           0 :     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << std::endl
     174           0 :            << "Expected: " << std::endl
     175           0 :            << writestr(expected) << std::endl
     176           0 :            << "Actual: " << std::endl
     177           0 :            << writestr(actual) << std::endl;
     178           0 :     if (!msg.empty())
     179           0 :         errmsg << "Hint:" << std::endl << msg << std::endl;
     180           0 :     errmsg << "_____" << std::endl;
     181           0 : }
     182             : 
     183       20613 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
     184             : {
     185       20613 :     if (expected != actual) {
     186           0 :         assertEqualsFailed(filename, linenr, expected, actual, msg);
     187             :     }
     188       20613 :     return expected == actual;
     189             : }
     190             : 
     191          20 : std::string TestFixture::deleteLineNumber(const std::string &message)
     192             : {
     193          20 :     std::string result(message);
     194             :     // delete line number in "...:NUMBER:..."
     195          20 :     std::string::size_type pos = 0;
     196         164 :     while ((pos = result.find(':', pos)) != std::string::npos) {
     197             :         // get number
     198         144 :         if (pos + 1 == result.find_first_of("0123456789", pos + 1)) {
     199          56 :             const std::string::size_type after = result.find_first_not_of("0123456789", pos + 1);
     200          56 :             if (after != std::string::npos
     201          56 :                 && result.at(after) == ':') {
     202             :                 // erase NUMBER
     203          22 :                 result.erase(pos + 1, after - pos - 1);
     204          22 :                 pos = after;
     205             :             } else {
     206          34 :                 ++pos;
     207             :             }
     208             :         } else {
     209          88 :             ++pos;
     210             :         }
     211             :     }
     212          20 :     return result;
     213             : }
     214             : 
     215          10 : void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
     216             : {
     217          10 :     assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg);
     218          10 : }
     219             : 
     220       19523 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const
     221             : {
     222       39046 :     return assertEquals(filename, linenr, std::string(expected), actual, msg);
     223             : }
     224          18 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const
     225             : {
     226          36 :     return assertEquals(filename, linenr, std::string(expected), std::string(actual), msg);
     227             : }
     228          27 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const
     229             : {
     230          54 :     return assertEquals(filename, linenr, expected, std::string(actual), msg);
     231             : }
     232             : 
     233        1270 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const
     234             : {
     235        1270 :     if (expected != actual) {
     236           0 :         assertEquals(filename, linenr, std::to_string(expected), std::to_string(actual), msg);
     237             :     }
     238        1270 :     return expected == actual;
     239             : }
     240             : 
     241          70 : void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const
     242             : {
     243          70 :     if (expected < (actual - tolerance) || expected > (actual + tolerance)) {
     244           2 :         std::ostringstream ostr1;
     245           1 :         ostr1 << expected;
     246           1 :         std::ostringstream ostr2;
     247           1 :         ostr2 << actual;
     248           1 :         assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
     249             :     }
     250          70 : }
     251             : 
     252         355 : void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr,
     253             :                                    const std::string &wanted,
     254             :                                    const std::string &current,
     255             :                                    const std::string &actual) const
     256             : {
     257         355 :     if (wanted == actual) {
     258           0 :         errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded unexpectedly. "
     259           0 :                << "Result: " << writestr(wanted, true)  << std::endl << "_____" << std::endl;
     260             : 
     261           0 :         ++succeeded_todos_counter;
     262             :     } else {
     263         355 :         assertEquals(filename, linenr, current, actual);
     264         355 :         ++todos_counter;
     265             :     }
     266         355 : }
     267             : 
     268         290 : void TestFixture::todoAssertEquals(const char* const filename, const unsigned int linenr,
     269             :                                    const char wanted[],
     270             :                                    const char current[],
     271             :                                    const std::string& actual) const
     272             : {
     273         290 :     todoAssertEquals(filename, linenr, std::string(wanted), std::string(current), actual);
     274         290 : }
     275             : 
     276             : 
     277          62 : void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const
     278             : {
     279          62 :     todoAssertEquals(filename, linenr, std::to_string(wanted), std::to_string(current), std::to_string(actual));
     280          62 : }
     281             : 
     282           0 : void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const
     283             : {
     284           0 :     ++fails_counter;
     285           0 :     errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
     286           0 :            << "The expected exception was thrown" << std::endl << "_____" << std::endl;
     287             : 
     288           0 : }
     289             : 
     290           0 : void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
     291             : {
     292           0 :     ++fails_counter;
     293           0 :     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
     294           0 :            << "The expected exception was not thrown"  << std::endl << "_____" << std::endl;
     295             : 
     296           0 : }
     297             : 
     298           0 : void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
     299             : {
     300           0 :     ++fails_counter;
     301             : 
     302           0 :     std::string ex_msg;
     303             : 
     304             :     try {
     305             :         // cppcheck-suppress rethrowNoCurrentException
     306           0 :         throw;
     307             :     }
     308           0 :     catch (const InternalError& e) {
     309           0 :         ex_msg = e.errorMessage;
     310             :     }
     311           0 :     catch (const std::exception& e) {
     312           0 :         ex_msg = e.what();
     313             :     }
     314           0 :     catch (...) {
     315           0 :         ex_msg = "unknown exception";
     316             :     }
     317             : 
     318           0 :     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
     319           0 :            << "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;
     320             : 
     321           0 : }
     322             : 
     323           0 : void TestFixture::printHelp()
     324             : {
     325             :     std::cout << "Testrunner - run Cppcheck tests\n"
     326             :         "\n"
     327             :         "Syntax:\n"
     328             :         "    testrunner [OPTIONS] [TestClass::TestCase...]\n"
     329             :         "    run all test cases:\n"
     330             :         "        testrunner\n"
     331             :         "    run all test cases in TestClass:\n"
     332             :         "        testrunner TestClass\n"
     333             :         "    run TestClass::TestCase:\n"
     334             :         "        testrunner TestClass::TestCase\n"
     335             :         "    run all test cases in TestClass1 and TestClass2::TestCase:\n"
     336             :         "        testrunner TestClass1 TestClass2::TestCase\n"
     337             :         "\n"
     338             :         "Options:\n"
     339             :         "    -q                   Do not print the test cases that have run.\n"
     340             :         "    -h, --help           Print this help.\n"
     341             :         "    -n                   Print no summaries.\n"
     342           0 :         "    -d                   Do not execute the tests.\n";
     343           0 : }
     344             : 
     345          77 : void TestFixture::run(const std::string &str)
     346             : {
     347          77 :     testToRun = str;
     348             :     try {
     349          77 :         if (quiet_tests) {
     350           0 :             std::cout << '\n' << classname << ':';
     351           0 :             SUPPRESS;
     352           0 :             run();
     353             :         }
     354             :         else
     355          77 :             run();
     356             :     }
     357           0 :     catch (const InternalError& e) {
     358           0 :         ++fails_counter;
     359           0 :         errmsg << classname << "::" << mTestname << " - InternalError: " << e.errorMessage << std::endl;
     360             :     }
     361           0 :     catch (const std::exception& error) {
     362           0 :         ++fails_counter;
     363           0 :         errmsg << classname << "::" << mTestname << " - Exception: " << error.what() << std::endl;
     364             :     }
     365           0 :     catch (...) {
     366           0 :         ++fails_counter;
     367           0 :         errmsg << classname << "::" << mTestname << " - Unknown exception" << std::endl;
     368             :     }
     369          77 : }
     370             : 
     371          77 : void TestFixture::processOptions(const options& args)
     372             : {
     373          77 :     quiet_tests = args.quiet();
     374          77 :     dry_run = args.dry_run();
     375          77 :     exename = args.exe();
     376          77 : }
     377             : 
     378           1 : std::size_t TestFixture::runTests(const options& args)
     379             : {
     380           1 :     countTests = 0;
     381           1 :     errmsg.str("");
     382             : 
     383           2 :     for (std::string classname : args.which_test()) {
     384           2 :         std::string testname;
     385           1 :         if (classname.find("::") != std::string::npos) {
     386           0 :             testname = classname.substr(classname.find("::") + 2);
     387           0 :             classname.erase(classname.find("::"));
     388             :         }
     389             : 
     390          78 :         for (TestFixture * test : TestRegistry::theInstance().tests()) {
     391          77 :             if (classname.empty() || test->classname == classname) {
     392          77 :                 test->processOptions(args);
     393          77 :                 test->run(testname);
     394             :             }
     395             :         }
     396             :     }
     397             : 
     398           1 :     if (args.summary() && !args.dry_run()) {
     399           1 :         std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
     400           1 :         std::cout << "Number of todos: " << todos_counter;
     401           1 :         if (succeeded_todos_counter > 0)
     402           0 :             std::cout << " (" << succeeded_todos_counter << " succeeded)";
     403           1 :         std::cout << std::endl;
     404             :     }
     405             :     // calling flush here, to do all output before the error messages (in case the output is buffered)
     406           1 :     std::cout.flush();
     407             : 
     408           1 :     if (args.summary() && !args.dry_run()) {
     409           1 :         std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
     410             :     }
     411           1 :     std::cerr << errmsg.str();
     412             : 
     413           1 :     std::cerr.flush();
     414           1 :     return fails_counter + succeeded_todos_counter;
     415             : }
     416             : 
     417         403 : void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
     418             : {
     419         403 :     mOutput << outmsg << std::endl;
     420         403 : }
     421             : 
     422      187188 : void TestFixture::reportErr(const ErrorMessage &msg)
     423             : {
     424      187188 :     if (msg.severity == Severity::internal)
     425      173012 :         return;
     426       14176 :     if (msg.severity == Severity::information && msg.id == "normalCheckLevelMaxBranches")
     427           0 :         return;
     428       28352 :     const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
     429       14176 :     mErrout << errormessage << std::endl;
     430             : }
     431             : 
     432          24 : void TestFixture::setTemplateFormat(const std::string &templateFormat)
     433             : {
     434          24 :     if (templateFormat == "multiline") {
     435          11 :         mTemplateFormat = "{file}:{line}:{severity}:{message}";
     436          11 :         mTemplateLocation = "{file}:{line}:note:{info}";
     437             :     }
     438          13 :     else if (templateFormat == "simple") { // TODO: use the existing one in CmdLineParser
     439          13 :         mTemplateFormat = "{file}:{line}:{column}: {severity}:{inconclusive:inconclusive:} {message} [{id}]";
     440          13 :         mTemplateLocation = "";
     441             :     }
     442             :     else {
     443           0 :         mTemplateFormat = templateFormat;
     444           0 :         mTemplateLocation = "";
     445             :     }
     446          24 : }
     447             : 
     448           0 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::checkLevel(Settings::CheckLevel level) {
     449           0 :     settings.setCheckLevel(level);
     450           0 :     return *this;
     451             : }
     452             : 
     453         549 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::library(const char lib[]) {
     454         549 :     if (REDUNDANT_CHECK && std::find(settings.libraries.cbegin(), settings.libraries.cend(), lib) != settings.libraries.cend())
     455           0 :         throw std::runtime_error("redundant setting: libraries (" + std::string(lib) + ")");
     456             :     // TODO: exename is not yet set
     457         549 :     LOAD_LIB_2_EXE(settings.library, lib, fixture.exename.c_str());
     458             :     // strip extension
     459        1098 :     std::string lib_s(lib);
     460         549 :     const std::string ext(".cfg");
     461         549 :     const auto pos = lib_s.find(ext);
     462         549 :     if (pos != std::string::npos)
     463         549 :         lib_s.erase(pos, ext.size());
     464         549 :     settings.libraries.emplace_back(lib_s);
     465        1098 :     return *this;
     466             : }
     467             : 
     468        2027 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::platform(Platform::Type type)
     469             : {
     470        4054 :     const std::string platformStr = Platform::toString(type);
     471             : 
     472        2027 :     if (REDUNDANT_CHECK && settings.platform.type == type)
     473           0 :         throw std::runtime_error("redundant setting: platform (" + platformStr + ")");
     474             : 
     475        2027 :     std::string errstr;
     476             :     // TODO: exename is not yet set
     477        4054 :     if (!settings.platform.set(platformStr, errstr, {fixture.exename}))
     478           0 :         throw std::runtime_error("platform '" + platformStr + "' not found");
     479        4054 :     return *this;
     480             : }
     481             : 
     482          47 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::libraryxml(const char xmldata[], std::size_t len)
     483             : {
     484          47 :     tinyxml2::XMLDocument doc;
     485          47 :     const tinyxml2::XMLError xml_error = doc.Parse(xmldata, len);
     486          47 :     if (tinyxml2::XML_SUCCESS != xml_error)
     487           0 :         throw std::runtime_error(std::string("loading XML data failed - ") + tinyxml2::XMLDocument::ErrorIDToName(xml_error));
     488          47 :     const Library::ErrorCode lib_error = settings.library.load(doc).errorcode;
     489          47 :     if (lib_error != Library::ErrorCode::OK)
     490           0 :         throw std::runtime_error("loading library XML failed - " + std::to_string(static_cast<int>(lib_error)));
     491          94 :     return *this;
     492             : }

Generated by: LCOV version 1.14