#include "tokenize.h"
#include "astutils.h"
#include "errorlogger.h"
#include "errortypes.h"
#include "library.h"
#include "mathlib.h"
#include "path.h"
#include "platform.h"
#include "preprocessor.h"
#include "settings.h"
#include "standards.h"
#include "summaries.h"
#include "symboldatabase.h"
#include "templatesimplifier.h"
#include "timer.h"
#include "token.h"
#include "utils.h"
#include "valueflow.h"
#include "vfvalue.h"
#include <algorithm>
#include <cassert>
#include <cctype>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <iostream>
#include <iterator>
#include <exception>
#include <memory>
#include <set>
#include <sstream>
#include <stack>
#include <stdexcept>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include <simplecpp.h>

Macros
#define	NOTSTART_C "NOT", "case", "default", "goto", "not", "return", "sizeof", "typedef"

Functions
static bool	isEnumStart (const Token *tok)
	Return whether tok is the "{" that starts an enumerator list. More...

template<typename T >
static void	skipEnumBody (T *&tok)

static bool	isClassStructUnionEnumStart (const Token *tok)
	is tok the start brace { of a class, struct, union, or enum More...

static Token *	splitDefinitionFromTypedef (Token tok, nonneg int unnamedCount)

static bool	isEnumScope (const Token *tok)

static bool	scopesMatch (const std::string &scope1, const std::string &scope2, const ScopeInfo3 *globalScope)

static unsigned int	tokDistance (const Token tok1, const Token tok2)

static Token *	skipTernaryOp (Token *tok)

static Token *	skipCaseLabel (Token *tok)

static bool	setVarIdParseDeclaration (Token *&tok, const VariableMap &variableMap, bool executableScope)

static void	setVarIdStructMembers (Token *&tok1, std::map< nonneg int, std::map< std::string, nonneg int >> &structMembers, nonneg int &varId)

static bool	setVarIdClassDeclaration (Token *const startToken, VariableMap &variableMap, const nonneg int scopeStartVarId, std::map< nonneg int, std::map< std::string, nonneg int >> &structMembers)

static std::string	getScopeName (const std::list< ScopeInfo2 > &scopeInfo)

static Token *	matchMemberName (const std::list< std::string > &scope, const Token nsToken, Token memberToken, const std::list< ScopeInfo2 > &scopeInfo)

static Token *	matchMemberName (const Member &member, const std::list< ScopeInfo2 > &scopeInfo)

static Token *	matchMemberVarName (const Member &var, const std::list< ScopeInfo2 > &scopeInfo)

static Token *	matchMemberFunctionName (const Member &func, const std::list< ScopeInfo2 > &scopeInfo)

template<typename T >
static T *	skipInitializerList (T *tok)

static void	linkBrackets (const Tokenizer &tokenizer, std::stack< const Token * > &type, std::stack< Token * > &links, Token *const token, const char open, const char close)

static std::string	getExpression (const Token *tok)

static bool	isNumberOneOf (const std::string &s, MathLib::bigint intConstant, const char *floatConstant)
	Helper function to check whether number is equal to integer constant X or floating point pattern X.0. More...

static const Token *	findUnmatchedTernaryOp (const Token const begin, const Token const end, int depth=0)

static bool	isCPPAttribute (const Token *tok)

static bool	isAlignAttribute (const Token *tok)

template<typename T >
static T *	skipCPPOrAlignAttribute (T *tok)

static bool	isNonMacro (const Token *tok)

Variables
static const std::unordered_set< std::string >	notstart_c = { NOTSTART_C }

static const std::unordered_set< std::string >	notstart_cpp

static const std::unordered_map< std::string, std::string >	cAlternativeTokens

Macro Definition Documentation

◆ NOTSTART_C

#define NOTSTART_C "NOT", "case", "default", "goto", "not", "return", "sizeof", "typedef"

Definition at line 4519 of file tokenize.cpp.

Function Documentation

◆ findUnmatchedTernaryOp()

static const Token* findUnmatchedTernaryOp	(	const Token *const	begin,
		const Token *const	end,
		int	depth = `0`
	)

static

Definition at line 8238 of file tokenize.cpp.

References Token::link(), Token::Match(), Token::next(), and Token::str().

◆ getExpression()

static std::string getExpression ( const Token * tok )

static

Definition at line 6329 of file tokenize.cpp.

References Token::Match(), Token::next(), Token::previous(), and Token::str().

Referenced by Tokenizer::splitTemplateRightAngleBrackets().

◆ getScopeName()

static std::string getScopeName ( const std::list< ScopeInfo2 > & scopeInfo )

static

Definition at line 4903 of file tokenize.cpp.

Referenced by Tokenizer::setVarIdPass2().

◆ isAlignAttribute()

static bool isAlignAttribute ( const Token * tok )

static

Definition at line 8261 of file tokenize.cpp.

References Token::link(), Token::next(), and Token::simpleMatch().

Referenced by skipCPPOrAlignAttribute().

◆ isClassStructUnionEnumStart()

static bool isClassStructUnionEnumStart ( const Token * tok )

static

is tok the start brace { of a class, struct, union, or enum

Definition at line 149 of file tokenize.cpp.

References Token::Match(), and Token::previous().

Referenced by Tokenizer::setVarIdPass1().

◆ isCPPAttribute()

static bool isCPPAttribute ( const Token * tok )

static

Definition at line 8256 of file tokenize.cpp.

References Token::link(), Token::linkAt(), and Token::simpleMatch().

Referenced by skipCPPOrAlignAttribute().

◆ isEnumScope()

static bool isEnumScope ( const Token * tok )

static

Definition at line 1135 of file tokenize.cpp.

References Token::isKeyword(), Token::Match(), Token::previous(), and Token::simpleMatch().

Referenced by Tokenizer::simplifyTypedefCpp().

◆ isEnumStart()

static bool isEnumStart ( const Token * tok )

static

Return whether tok is the "{" that starts an enumerator list.

Definition at line 80 of file tokenize.cpp.

References Token::Match(), Token::str(), Token::strAt(), and Token::tokAt().

Referenced by setVarIdClassDeclaration(), and Tokenizer::setVarIdPass1().

◆ isNonMacro()

static bool isNonMacro ( const Token * tok )

static

Definition at line 8277 of file tokenize.cpp.

References cAlternativeTokens, Token::isKeyword(), Token::isStandardType(), Token::simpleMatch(), startsWith(), and Token::str().

Referenced by Tokenizer::reportUnknownMacros().

◆ isNumberOneOf()

static bool isNumberOneOf	(	const std::string &	s,
		MathLib::bigint	intConstant,
		const char *	floatConstant
	)

static

Helper function to check whether number is equal to integer constant X or floating point pattern X.0.

Parameters

s	the string to check
intConstant	the integer constant to check against
floatConstant	the string with stringified float constant to check against

Returns: true in case s is equal to X or X.0 and false otherwise.

Definition at line 8124 of file tokenize.cpp.

References MathLib::isFloat(), MathLib::isInt(), MathLib::toBigNumber(), MathLib::toDoubleNumber(), and MathLib::toString().

Referenced by Tokenizer::isOneNumber().

◆ linkBrackets()

static void linkBrackets	(	const Tokenizer &	tokenizer,
		std::stack< const Token * > &	type,
		std::stack< Token * > &	links,
		Token *const	token,
		const char	open,
		const char	close
	)

static

Definition at line 5257 of file tokenize.cpp.

References Token::createMutualLinks(), Token::str(), and Tokenizer::unmatchedToken().

Referenced by Tokenizer::createLinks().

◆ matchMemberFunctionName()

static Token* matchMemberFunctionName	(	const Member &	func,
		const std::list< ScopeInfo2 > &	scopeInfo
	)

static

Definition at line 4987 of file tokenize.cpp.

References Token::Match(), and matchMemberName().

Referenced by Tokenizer::setVarIdPass2().

◆ matchMemberName() [1/2]

static Token* matchMemberName	(	const Member &	member,
		const std::list< ScopeInfo2 > &	scopeInfo
	)

static

Definition at line 4957 of file tokenize.cpp.

References matchMemberName().

◆ matchMemberName() [2/2]

static Token* matchMemberName	(	const std::list< std::string > &	scope,
		const Token *	nsToken,
		Token *	memberToken,
		const std::list< ScopeInfo2 > &	scopeInfo
	)

static

Definition at line 4911 of file tokenize.cpp.

References Token::findClosingBracket(), Token::Match(), Token::next(), Token::simpleMatch(), Token::str(), and Token::tokAt().

Referenced by matchMemberFunctionName(), matchMemberName(), and matchMemberVarName().

◆ matchMemberVarName()

static Token* matchMemberVarName	(	const Member &	var,
		const std::list< ScopeInfo2 > &	scopeInfo
	)

static

Definition at line 4977 of file tokenize.cpp.

References Token::isLiteral(), Token::Match(), matchMemberName(), Token::next(), Token::strAt(), and Token::tokAt().

Referenced by Tokenizer::setVarIdPass2().

◆ scopesMatch()

static bool scopesMatch	(	const std::string &	scope1,
		const std::string &	scope2,
		const ScopeInfo3 *	globalScope
	)

static

Definition at line 2819 of file tokenize.cpp.

Referenced by Tokenizer::simplifyUsing().

◆ setVarIdClassDeclaration()

static bool setVarIdClassDeclaration	(	Token *const	startToken,
		VariableMap &	variableMap,
		const nonneg int	scopeStartVarId,
		std::map< nonneg int, std::map< std::string, nonneg int >> &	structMembers
	)

static

Definition at line 4392 of file tokenize.cpp.

References isEnumStart(), Token::isName(), Token::link(), Token::Match(), Token::next(), Token::previous(), setVarIdStructMembers(), Token::simpleMatch(), Token::str(), Token::strAt(), Token::tokAt(), and Token::varId().

Referenced by Tokenizer::setVarIdPass1().

◆ setVarIdParseDeclaration()

static bool setVarIdParseDeclaration	(	Token *&	tok,
		const VariableMap &	variableMap,
		bool	executableScope
	)

static

Definition at line 4185 of file tokenize.cpp.

References Token::findClosingBracket(), Token::isCpp(), Token::isName(), Token::isStandardType(), Token::link(), Token::linkAt(), Token::Match(), Token::next(), nonneg, Token::previous(), Token::simpleMatch(), Token::str(), TemplateSimplifier::templateParameters(), and Token::tokAt().

Referenced by Tokenizer::setVarIdPass1().

◆ setVarIdStructMembers()

static void setVarIdStructMembers	(	Token *&	tok1,
		std::map< nonneg int, std::map< std::string, nonneg int >> &	structMembers,
		nonneg int &	varId
	)

static

Definition at line 4327 of file tokenize.cpp.

References Tokenizer::isFunctionHead(), Token::link(), Token::linkAt(), Token::Match(), Token::next(), nonneg, Token::previous(), Token::str(), Token::strAt(), TemplateSimplifier::templateParameters(), Token::tokAt(), and Token::varId().

Referenced by setVarIdClassDeclaration(), Tokenizer::setVarIdClassFunction(), and Tokenizer::setVarIdPass1().

◆ skipCaseLabel()

static Token* skipCaseLabel ( Token * tok )

static

Definition at line 3908 of file tokenize.cpp.

References Token::link(), Token::Match(), Token::next(), skipTernaryOp(), and Token::str().

Referenced by Tokenizer::simplifyAddBracesPair(), and Tokenizer::simplifyLabelsCaseDefault().

◆ skipCPPOrAlignAttribute()

template<typename T >

static T* skipCPPOrAlignAttribute ( T * tok )

static

Definition at line 8267 of file tokenize.cpp.

References isAlignAttribute(), and isCPPAttribute().

◆ skipEnumBody()

template<typename T >

static void skipEnumBody ( T *& tok )

static

Definition at line 88 of file tokenize.cpp.

References Token::Match().

◆ skipInitializerList()

template<typename T >

static T* skipInitializerList ( T * tok )

static

Definition at line 4994 of file tokenize.cpp.

References Token::Match(), Token::str(), and Token::tokAt().

Referenced by Tokenizer::setVarIdPass2().

◆ skipTernaryOp()

static Token* skipTernaryOp ( Token * tok )

static

Definition at line 3882 of file tokenize.cpp.

References Token::link(), Token::Match(), Token::next(), and Token::str().

Referenced by skipCaseLabel().

◆ splitDefinitionFromTypedef()

static Token* splitDefinitionFromTypedef	(	Token *	tok,
		nonneg int *	unnamedCount
	)

static

Definition at line 407 of file tokenize.cpp.

References Token::deleteNext(), Token::deleteThis(), Token::insertToken(), Token::link(), Token::Match(), Token::next(), Token::previous(), Token::str(), and Token::tokAt().

Referenced by Tokenizer::simplifyTypedefCpp().

◆ tokDistance()

static unsigned int tokDistance	(	const Token *	tok1,
		const Token *	tok2
	)

static

Definition at line 2846 of file tokenize.cpp.

References Token::next().

Referenced by Tokenizer::simplifyUsing().

Variable Documentation

◆ cAlternativeTokens

const std::unordered_map<std::string, std::string> cAlternativeTokens

static

Initial value:

= {
    std::make_pair("and", "&&")
    , std::make_pair("and_eq", "&=")
    , std::make_pair("bitand", "&")
    , std::make_pair("bitor", "|")
    , std::make_pair("not_eq", "!=")
    , std::make_pair("or", "||")
    , std::make_pair("or_eq", "|=")
    , std::make_pair("xor", "^")
    , std::make_pair("xor_eq", "^=")
}

Definition at line 7513 of file tokenize.cpp.

Referenced by isNonMacro(), Tokenizer::reportUnknownMacros(), and Tokenizer::simplifyCAlternativeTokens().

◆ notstart_c

const std::unordered_set<std::string> notstart_c = { NOTSTART_C }

static

Definition at line 4520 of file tokenize.cpp.

Referenced by Tokenizer::setVarIdPass1().

◆ notstart_cpp

const std::unordered_set<std::string> notstart_cpp

static

Initial value:

= { NOTSTART_C,
                                                              "delete", "friend", "new", "throw", "using", "virtual", "explicit", "const_cast", "dynamic_cast", "reinterpret_cast", "static_cast", "template"
}

Definition at line 4521 of file tokenize.cpp.

Referenced by Tokenizer::setVarIdPass1().

Macros

Functions

Variables

Macro Definition Documentation

◆ NOTSTART_C

Function Documentation

◆ findUnmatchedTernaryOp()

◆ getExpression()

◆ getScopeName()

◆ isAlignAttribute()

◆ isClassStructUnionEnumStart()

◆ isCPPAttribute()

◆ isEnumScope()

◆ isEnumStart()

◆ isNonMacro()

◆ isNumberOneOf()

◆ linkBrackets()

◆ matchMemberFunctionName()

◆ matchMemberName() [1/2]

◆ matchMemberName() [2/2]

◆ matchMemberVarName()

◆ scopesMatch()

◆ setVarIdClassDeclaration()

◆ setVarIdParseDeclaration()

◆ setVarIdStructMembers()

◆ skipCaseLabel()

◆ skipCPPOrAlignAttribute()

◆ skipEnumBody()

◆ skipInitializerList()

◆ skipTernaryOp()

◆ splitDefinitionFromTypedef()

◆ tokDistance()

Variable Documentation

◆ cAlternativeTokens

◆ notstart_c

◆ notstart_cpp