Line data Source code
1 : /*
2 : * Cppcheck - A tool for static C/C++ code analysis
3 : * Copyright (C) 2007-2024 Cppcheck team.
4 : *
5 : * This program is free software: you can redistribute it and/or modify
6 : * it under the terms of the GNU General Public License as published by
7 : * the Free Software Foundation, either version 3 of the License, or
8 : * (at your option) any later version.
9 : *
10 : * This program is distributed in the hope that it will be useful,
11 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : * GNU General Public License for more details.
14 : *
15 : * You should have received a copy of the GNU General Public License
16 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 : */
18 :
19 : #include "fixture.h"
20 :
21 : #include "cppcheck.h"
22 : #include "errortypes.h"
23 : #include "options.h"
24 : #include "redirect.h"
25 :
26 : #include <algorithm>
27 : #include <cstdio>
28 : #include <cctype>
29 : #include <exception>
30 : #include <iostream>
31 : #include <set>
32 : #include <sstream>
33 : #include <string>
34 :
35 : #include "xml.h"
36 :
37 : /**
38 : * TestRegistry
39 : **/
40 : namespace {
41 : struct CompareFixtures {
42 714 : bool operator()(const TestFixture* lhs, const TestFixture* rhs) const {
43 714 : return lhs->classname < rhs->classname;
44 : }
45 : };
46 : }
47 : using TestSet = std::set<TestFixture*, CompareFixtures>;
48 : namespace {
49 : class TestRegistry {
50 : TestSet _tests;
51 : public:
52 :
53 78 : static TestRegistry &theInstance() {
54 78 : static TestRegistry testreg;
55 78 : return testreg;
56 : }
57 :
58 77 : void addTest(TestFixture *t) {
59 77 : _tests.insert(t);
60 77 : }
61 :
62 1 : const TestSet &tests() const {
63 1 : return _tests;
64 : }
65 : };
66 : }
67 :
68 :
69 :
70 :
71 : /**
72 : * TestFixture
73 : **/
74 :
75 : std::ostringstream TestFixture::errmsg;
76 : unsigned int TestFixture::countTests;
77 :
78 : std::size_t TestFixture::fails_counter = 0;
79 : std::size_t TestFixture::todos_counter = 0;
80 : std::size_t TestFixture::succeeded_todos_counter = 0;
81 :
82 77 : TestFixture::TestFixture(const char * const _name)
83 77 : : classname(_name)
84 : {
85 77 : TestRegistry::theInstance().addTest(this);
86 77 : }
87 :
88 :
89 4701 : bool TestFixture::prepareTest(const char testname[])
90 : {
91 4701 : mVerbose = false;
92 4701 : mTemplateFormat.clear();
93 4701 : mTemplateLocation.clear();
94 4701 : CppCheck::resetTimerResults();
95 :
96 4701 : prepareTestInternal();
97 :
98 : // Check if tests should be executed
99 4701 : if (testToRun.empty() || testToRun == testname) {
100 : // Tests will be executed - prepare them
101 4701 : mTestname = testname;
102 4701 : ++countTests;
103 4701 : if (quiet_tests) {
104 0 : std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
105 0 : std::fflush(stdout);
106 : } else {
107 4701 : std::cout << classname << "::" << mTestname << std::endl;
108 : }
109 4701 : return !dry_run;
110 : }
111 0 : return false;
112 : }
113 :
114 4701 : void TestFixture::teardownTest()
115 : {
116 4701 : teardownTestInternal();
117 :
118 : {
119 9402 : const std::string s = errout_str();
120 4701 : if (!s.empty())
121 0 : throw std::runtime_error("unconsumed ErrorLogger err: " + s);
122 : }
123 : {
124 9402 : const std::string s = output_str();
125 4701 : if (!s.empty())
126 0 : throw std::runtime_error("unconsumed ErrorLogger out: " + s);
127 : }
128 4701 : }
129 :
130 0 : std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
131 : {
132 0 : return std::string(filename) + ':' + std::to_string(linenr) + '(' + classname + "::" + mTestname + ')';
133 : }
134 :
135 0 : static std::string writestr(const std::string &str, bool gccStyle = false)
136 : {
137 0 : std::ostringstream ostr;
138 0 : if (gccStyle)
139 0 : ostr << '\"';
140 0 : for (std::string::const_iterator i = str.cbegin(); i != str.cend(); ++i) {
141 0 : if (*i == '\n') {
142 0 : ostr << "\\n";
143 0 : if ((i+1) != str.end() && !gccStyle)
144 0 : ostr << std::endl;
145 0 : } else if (*i == '\t')
146 0 : ostr << "\\t";
147 0 : else if (*i == '\"')
148 0 : ostr << "\\\"";
149 0 : else if (std::isprint(static_cast<unsigned char>(*i)))
150 0 : ostr << *i;
151 : else
152 0 : ostr << "\\x" << std::hex << short{*i};
153 : }
154 0 : if (!str.empty() && !gccStyle)
155 0 : ostr << std::endl;
156 0 : else if (gccStyle)
157 0 : ostr << '\"';
158 0 : return ostr.str();
159 : }
160 :
161 22634 : bool TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const
162 : {
163 22634 : if (!condition) {
164 0 : ++fails_counter;
165 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
166 : }
167 22634 : return condition;
168 : }
169 :
170 0 : void TestFixture::assertEqualsFailed(const char* const filename, const unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const
171 : {
172 0 : ++fails_counter;
173 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << std::endl
174 0 : << "Expected: " << std::endl
175 0 : << writestr(expected) << std::endl
176 0 : << "Actual: " << std::endl
177 0 : << writestr(actual) << std::endl;
178 0 : if (!msg.empty())
179 0 : errmsg << "Hint:" << std::endl << msg << std::endl;
180 0 : errmsg << "_____" << std::endl;
181 0 : }
182 :
183 20613 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
184 : {
185 20613 : if (expected != actual) {
186 0 : assertEqualsFailed(filename, linenr, expected, actual, msg);
187 : }
188 20613 : return expected == actual;
189 : }
190 :
191 20 : std::string TestFixture::deleteLineNumber(const std::string &message)
192 : {
193 20 : std::string result(message);
194 : // delete line number in "...:NUMBER:..."
195 20 : std::string::size_type pos = 0;
196 164 : while ((pos = result.find(':', pos)) != std::string::npos) {
197 : // get number
198 144 : if (pos + 1 == result.find_first_of("0123456789", pos + 1)) {
199 56 : const std::string::size_type after = result.find_first_not_of("0123456789", pos + 1);
200 56 : if (after != std::string::npos
201 56 : && result.at(after) == ':') {
202 : // erase NUMBER
203 22 : result.erase(pos + 1, after - pos - 1);
204 22 : pos = after;
205 : } else {
206 34 : ++pos;
207 : }
208 : } else {
209 88 : ++pos;
210 : }
211 : }
212 20 : return result;
213 : }
214 :
215 10 : void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
216 : {
217 10 : assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg);
218 10 : }
219 :
220 19523 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const
221 : {
222 39046 : return assertEquals(filename, linenr, std::string(expected), actual, msg);
223 : }
224 18 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const
225 : {
226 36 : return assertEquals(filename, linenr, std::string(expected), std::string(actual), msg);
227 : }
228 27 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const
229 : {
230 54 : return assertEquals(filename, linenr, expected, std::string(actual), msg);
231 : }
232 :
233 1270 : bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const
234 : {
235 1270 : if (expected != actual) {
236 0 : assertEquals(filename, linenr, std::to_string(expected), std::to_string(actual), msg);
237 : }
238 1270 : return expected == actual;
239 : }
240 :
241 70 : void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const
242 : {
243 70 : if (expected < (actual - tolerance) || expected > (actual + tolerance)) {
244 2 : std::ostringstream ostr1;
245 1 : ostr1 << expected;
246 1 : std::ostringstream ostr2;
247 1 : ostr2 << actual;
248 1 : assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
249 : }
250 70 : }
251 :
252 355 : void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr,
253 : const std::string &wanted,
254 : const std::string ¤t,
255 : const std::string &actual) const
256 : {
257 355 : if (wanted == actual) {
258 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded unexpectedly. "
259 0 : << "Result: " << writestr(wanted, true) << std::endl << "_____" << std::endl;
260 :
261 0 : ++succeeded_todos_counter;
262 : } else {
263 355 : assertEquals(filename, linenr, current, actual);
264 355 : ++todos_counter;
265 : }
266 355 : }
267 :
268 290 : void TestFixture::todoAssertEquals(const char* const filename, const unsigned int linenr,
269 : const char wanted[],
270 : const char current[],
271 : const std::string& actual) const
272 : {
273 290 : todoAssertEquals(filename, linenr, std::string(wanted), std::string(current), actual);
274 290 : }
275 :
276 :
277 62 : void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const
278 : {
279 62 : todoAssertEquals(filename, linenr, std::to_string(wanted), std::to_string(current), std::to_string(actual));
280 62 : }
281 :
282 0 : void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const
283 : {
284 0 : ++fails_counter;
285 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
286 0 : << "The expected exception was thrown" << std::endl << "_____" << std::endl;
287 :
288 0 : }
289 :
290 0 : void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
291 : {
292 0 : ++fails_counter;
293 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
294 0 : << "The expected exception was not thrown" << std::endl << "_____" << std::endl;
295 :
296 0 : }
297 :
298 0 : void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
299 : {
300 0 : ++fails_counter;
301 :
302 0 : std::string ex_msg;
303 :
304 : try {
305 : // cppcheck-suppress rethrowNoCurrentException
306 0 : throw;
307 : }
308 0 : catch (const InternalError& e) {
309 0 : ex_msg = e.errorMessage;
310 : }
311 0 : catch (const std::exception& e) {
312 0 : ex_msg = e.what();
313 : }
314 0 : catch (...) {
315 0 : ex_msg = "unknown exception";
316 : }
317 :
318 0 : errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
319 0 : << "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;
320 :
321 0 : }
322 :
323 0 : void TestFixture::printHelp()
324 : {
325 : std::cout << "Testrunner - run Cppcheck tests\n"
326 : "\n"
327 : "Syntax:\n"
328 : " testrunner [OPTIONS] [TestClass::TestCase...]\n"
329 : " run all test cases:\n"
330 : " testrunner\n"
331 : " run all test cases in TestClass:\n"
332 : " testrunner TestClass\n"
333 : " run TestClass::TestCase:\n"
334 : " testrunner TestClass::TestCase\n"
335 : " run all test cases in TestClass1 and TestClass2::TestCase:\n"
336 : " testrunner TestClass1 TestClass2::TestCase\n"
337 : "\n"
338 : "Options:\n"
339 : " -q Do not print the test cases that have run.\n"
340 : " -h, --help Print this help.\n"
341 : " -n Print no summaries.\n"
342 0 : " -d Do not execute the tests.\n";
343 0 : }
344 :
345 77 : void TestFixture::run(const std::string &str)
346 : {
347 77 : testToRun = str;
348 : try {
349 77 : if (quiet_tests) {
350 0 : std::cout << '\n' << classname << ':';
351 0 : SUPPRESS;
352 0 : run();
353 : }
354 : else
355 77 : run();
356 : }
357 0 : catch (const InternalError& e) {
358 0 : ++fails_counter;
359 0 : errmsg << classname << "::" << mTestname << " - InternalError: " << e.errorMessage << std::endl;
360 : }
361 0 : catch (const std::exception& error) {
362 0 : ++fails_counter;
363 0 : errmsg << classname << "::" << mTestname << " - Exception: " << error.what() << std::endl;
364 : }
365 0 : catch (...) {
366 0 : ++fails_counter;
367 0 : errmsg << classname << "::" << mTestname << " - Unknown exception" << std::endl;
368 : }
369 77 : }
370 :
371 77 : void TestFixture::processOptions(const options& args)
372 : {
373 77 : quiet_tests = args.quiet();
374 77 : dry_run = args.dry_run();
375 77 : exename = args.exe();
376 77 : }
377 :
378 1 : std::size_t TestFixture::runTests(const options& args)
379 : {
380 1 : countTests = 0;
381 1 : errmsg.str("");
382 :
383 2 : for (std::string classname : args.which_test()) {
384 2 : std::string testname;
385 1 : if (classname.find("::") != std::string::npos) {
386 0 : testname = classname.substr(classname.find("::") + 2);
387 0 : classname.erase(classname.find("::"));
388 : }
389 :
390 78 : for (TestFixture * test : TestRegistry::theInstance().tests()) {
391 77 : if (classname.empty() || test->classname == classname) {
392 77 : test->processOptions(args);
393 77 : test->run(testname);
394 : }
395 : }
396 : }
397 :
398 1 : if (args.summary() && !args.dry_run()) {
399 1 : std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
400 1 : std::cout << "Number of todos: " << todos_counter;
401 1 : if (succeeded_todos_counter > 0)
402 0 : std::cout << " (" << succeeded_todos_counter << " succeeded)";
403 1 : std::cout << std::endl;
404 : }
405 : // calling flush here, to do all output before the error messages (in case the output is buffered)
406 1 : std::cout.flush();
407 :
408 1 : if (args.summary() && !args.dry_run()) {
409 1 : std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
410 : }
411 1 : std::cerr << errmsg.str();
412 :
413 1 : std::cerr.flush();
414 1 : return fails_counter + succeeded_todos_counter;
415 : }
416 :
417 403 : void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
418 : {
419 403 : mOutput << outmsg << std::endl;
420 403 : }
421 :
422 187188 : void TestFixture::reportErr(const ErrorMessage &msg)
423 : {
424 187188 : if (msg.severity == Severity::internal)
425 173012 : return;
426 14176 : if (msg.severity == Severity::information && msg.id == "normalCheckLevelMaxBranches")
427 0 : return;
428 28352 : const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
429 14176 : mErrout << errormessage << std::endl;
430 : }
431 :
432 24 : void TestFixture::setTemplateFormat(const std::string &templateFormat)
433 : {
434 24 : if (templateFormat == "multiline") {
435 11 : mTemplateFormat = "{file}:{line}:{severity}:{message}";
436 11 : mTemplateLocation = "{file}:{line}:note:{info}";
437 : }
438 13 : else if (templateFormat == "simple") { // TODO: use the existing one in CmdLineParser
439 13 : mTemplateFormat = "{file}:{line}:{column}: {severity}:{inconclusive:inconclusive:} {message} [{id}]";
440 13 : mTemplateLocation = "";
441 : }
442 : else {
443 0 : mTemplateFormat = templateFormat;
444 0 : mTemplateLocation = "";
445 : }
446 24 : }
447 :
448 0 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::checkLevel(Settings::CheckLevel level) {
449 0 : settings.setCheckLevel(level);
450 0 : return *this;
451 : }
452 :
453 549 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::library(const char lib[]) {
454 549 : if (REDUNDANT_CHECK && std::find(settings.libraries.cbegin(), settings.libraries.cend(), lib) != settings.libraries.cend())
455 0 : throw std::runtime_error("redundant setting: libraries (" + std::string(lib) + ")");
456 : // TODO: exename is not yet set
457 549 : LOAD_LIB_2_EXE(settings.library, lib, fixture.exename.c_str());
458 : // strip extension
459 1098 : std::string lib_s(lib);
460 549 : const std::string ext(".cfg");
461 549 : const auto pos = lib_s.find(ext);
462 549 : if (pos != std::string::npos)
463 549 : lib_s.erase(pos, ext.size());
464 549 : settings.libraries.emplace_back(lib_s);
465 1098 : return *this;
466 : }
467 :
468 2027 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::platform(Platform::Type type)
469 : {
470 4054 : const std::string platformStr = Platform::toString(type);
471 :
472 2027 : if (REDUNDANT_CHECK && settings.platform.type == type)
473 0 : throw std::runtime_error("redundant setting: platform (" + platformStr + ")");
474 :
475 2027 : std::string errstr;
476 : // TODO: exename is not yet set
477 4054 : if (!settings.platform.set(platformStr, errstr, {fixture.exename}))
478 0 : throw std::runtime_error("platform '" + platformStr + "' not found");
479 4054 : return *this;
480 : }
481 :
482 47 : TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::libraryxml(const char xmldata[], std::size_t len)
483 : {
484 47 : tinyxml2::XMLDocument doc;
485 47 : const tinyxml2::XMLError xml_error = doc.Parse(xmldata, len);
486 47 : if (tinyxml2::XML_SUCCESS != xml_error)
487 0 : throw std::runtime_error(std::string("loading XML data failed - ") + tinyxml2::XMLDocument::ErrorIDToName(xml_error));
488 47 : const Library::ErrorCode lib_error = settings.library.load(doc).errorcode;
489 47 : if (lib_error != Library::ErrorCode::OK)
490 0 : throw std::runtime_error("loading library XML failed - " + std::to_string(static_cast<int>(lib_error)));
491 94 : return *this;
492 : }
|
