Skip to content

Commit ebc80c0

Browse files
firewavefirewave
authored
pass language to TokenList constructor (#7500)
1 parent 3726ace commit ebc80c0

20 files changed

+107
-151
lines changed

‎lib/clangimport.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,8 +677,7 @@ void clangimport::AstNode::setValueType(Token *tok)
677677
// TODO
678678
continue;
679679

680-
TokenList decl(nullptr);
681-
decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
680+
TokenList decl(nullptr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
682681
addTypeTokens(decl, type, tok->scope());
683682
if (!decl.front())
684683
break;

‎lib/cppcheck.cpp

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,7 @@ unsigned int CppCheck::checkClang(const FileWithDetails &file)
715715
}
716716

717717
try {
718-
TokenList tokenlist{&mSettings};
718+
TokenList tokenlist{&mSettings, file.lang()};
719719
tokenlist.appendFileIfNew(file.spath());
720720
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
721721
std::istringstream ast(output2);
@@ -910,9 +910,8 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
910910

911911
if (mUnusedFunctionsCheck && (mSettings.useSingleJob() || analyzerInformation)) {
912912
std::size_t hash = 0;
913-
TokenList tokenlist{&mSettings};
914-
// enforce the language since markup files are special and do not adhere to the enforced language
915-
tokenlist.setLang(Standards::Language::C, true);
913+
// markup files are special and do not adhere to the enforced language
914+
TokenList tokenlist{&mSettings, Standards::Language::C};
916915
if (fileStream) {
917916
std::vector<std::string> files;
918917
simplecpp::TokenList tokens(*fileStream, files, file.spath());
@@ -1056,10 +1055,9 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
10561055
if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include "))
10571056
code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
10581057
}
1059-
TokenList tokenlist(&mSettings);
1058+
TokenList tokenlist(&mSettings, file.lang());
10601059
std::istringstream istr2(code);
1061-
// TODO: asserts when file has unknown extension
1062-
tokenlist.createTokens(istr2, Path::identify(*files.begin(), false)); // TODO: check result?
1060+
tokenlist.createTokens(istr2); // TODO: check result?
10631061
executeRules("define", tokenlist);
10641062
}
10651063
#endif
@@ -1135,7 +1133,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
11351133
}
11361134

11371135
try {
1138-
TokenList tokenlist{&mSettings};
1136+
TokenList tokenlist{&mSettings, file.lang()};
11391137

11401138
// Create tokens, skip rest of iteration if failed
11411139
Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() {

‎lib/importproject.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -577,9 +577,9 @@ namespace {
577577

578578
// TODO: improve evaluation
579579
const Settings s;
580-
TokenList tokenlist(&s);
580+
TokenList tokenlist(&s, Standards::Language::C);
581581
std::istringstream istr(c);
582-
tokenlist.createTokens(istr, Standards::Language::C); // TODO: check result
582+
tokenlist.createTokens(istr); // TODO: check result
583583
// TODO: put in a helper
584584
// generate links
585585
{

‎lib/library.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -175,10 +175,10 @@ static std::vector<std::string> getnames(const char *names)
175175
return ret;
176176
}
177177

178-
static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
178+
static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
179179
{
180180
std::istringstream istr(valid + ',');
181-
tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
181+
tokenList.createTokens(istr); // TODO: check result?
182182
for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
183183
if (Token::Match(tok,"- %num%")) {
184184
tok->str("-" + tok->strAt(1));
@@ -1062,8 +1062,8 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
10621062
return true;
10631063
if (ac->valid.find('.') != std::string::npos)
10641064
return isFloatArgValid(ftok, argnr, static_cast<double>(argvalue));
1065-
TokenList tokenList(nullptr);
1066-
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
1065+
TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
1066+
gettokenlistfromvalid(ac->valid, tokenList);
10671067
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
10681068
if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok))
10691069
return true;
@@ -1082,8 +1082,8 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
10821082
const ArgumentChecks *ac = getarg(ftok, argnr);
10831083
if (!ac || ac->valid.empty())
10841084
return true;
1085-
TokenList tokenList(nullptr);
1086-
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
1085+
TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
1086+
gettokenlistfromvalid(ac->valid, tokenList);
10871087
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
10881088
if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok) && argvalue <= MathLib::toDoubleNumber(tok->tokAt(2)))
10891089
return true;

‎lib/programmemory.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1750,11 +1750,11 @@ static std::shared_ptr<Token> createTokenFromExpression(const std::string& retur
17501750
bool cpp,
17511751
std::unordered_map<nonneg int, const Token*>& lookupVarId)
17521752
{
1753-
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(&settings);
1753+
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(&settings, cpp ? Standards::Language::CPP : Standards::Language::C);
17541754
{
17551755
const std::string code = "return " + returnValue + ";";
17561756
std::istringstream istr(code);
1757-
if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
1757+
if (!tokenList->createTokens(istr))
17581758
return nullptr;
17591759
}
17601760

‎lib/symboldatabase.cpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1808,8 +1808,7 @@ void SymbolDatabase::setArrayDimensionsUsingValueFlow()
18081808

18091809
// In template arguments, there might not be AST
18101810
// Determine size by using the "raw tokens"
1811-
TokenList tokenList(&mSettings);
1812-
tokenList.setLang(dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
1811+
TokenList tokenList(&mSettings, dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
18131812
tokenList.addtoken(";", 0, 0, 0, false);
18141813
bool fail = false;
18151814
for (const Token *tok = dimension.tok; tok && !Token::Match(tok, "[,>]"); tok = tok->next()) {
@@ -7293,7 +7292,7 @@ static const Token* parsedecl(const Token* type,
72937292
else if (Token::simpleMatch(type, "volatile"))
72947293
valuetype->volatileness |= (1 << (valuetype->pointer - pointer0));
72957294
else if (settings.clang && type->str().size() > 2 && type->str().find("::") < type->str().find('<')) {
7296-
TokenList typeTokens(&settings);
7295+
TokenList typeTokens(&settings, type->isCpp() ? Standards::Language::CPP : Standards::Language::C);
72977296
std::string::size_type pos1 = 0;
72987297
do {
72997298
const std::string::size_type pos2 = type->str().find("::", pos1);
@@ -7716,9 +7715,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
77167715
const std::string& typestr(mSettings.library.returnValueType(tok->previous()));
77177716
if (!typestr.empty()) {
77187717
ValueType valuetype;
7719-
TokenList tokenList(&mSettings);
7718+
TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
77207719
std::istringstream istr(typestr+";");
7721-
tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
7720+
tokenList.createTokens(istr); // TODO: check result?
77227721
tokenList.simplifyStdType();
77237722
if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings)) {
77247723
valuetype.originalTypeName = typestr;
@@ -7806,9 +7805,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
78067805
}
78077806
continue;
78087807
}
7809-
TokenList tokenList(&mSettings);
7808+
TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
78107809
std::istringstream istr(typestr+";");
7811-
if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
7810+
if (tokenList.createTokens(istr)) {
78127811
ValueType vt;
78137812
tokenList.simplifyPlatformTypes();
78147813
tokenList.simplifyStdType();

‎lib/tokenize.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10870,7 +10870,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
1087010870

1087110871
void Tokenizer::getErrorMessages(ErrorLogger& errorLogger, const Settings& settings)
1087210872
{
10873-
TokenList tokenlist{&settings};
10873+
TokenList tokenlist{&settings, Standards::Language::C};
1087410874
Tokenizer tokenizer(std::move(tokenlist), settings, errorLogger);
1087510875
tokenizer.invalidConstFunctionTypeError(nullptr);
1087610876
// checkLibraryNoReturn

‎lib/tokenlist.cpp

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,12 @@
6060
static constexpr int AST_MAX_DEPTH = 150;
6161

6262

63-
TokenList::TokenList(const Settings* settings)
63+
TokenList::TokenList(const Settings* settings, Standards::Language lang)
6464
: mTokensFrontBack(new TokensFrontBack)
6565
, mSettings(settings)
6666
{
67-
if (mSettings && (mSettings->enforcedLang != Standards::Language::None)) {
68-
mLang = mSettings->enforcedLang;
69-
}
67+
assert(lang != Standards::Language::None);
68+
mLang = lang;
7069
}
7170

7271
TokenList::~TokenList()
@@ -346,15 +345,8 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
346345

347346
//---------------------------------------------------------------------------
348347

349-
bool TokenList::createTokens(std::istream &code, Standards::Language lang)
348+
bool TokenList::createTokens(std::istream &code)
350349
{
351-
ASSERT_LANG(lang != Standards::Language::None);
352-
if (mLang == Standards::Language::None) {
353-
mLang = lang;
354-
} else {
355-
ASSERT_LANG(lang == mLang);
356-
}
357-
358350
return createTokensInternal(code, mFiles.empty() ? "" : *mFiles.cbegin());
359351
}
360352

@@ -2288,17 +2280,6 @@ bool TokenList::isCPP() const
22882280
return mLang == Standards::Language::CPP;
22892281
}
22902282

2291-
void TokenList::setLang(Standards::Language lang, bool force)
2292-
{
2293-
ASSERT_LANG(lang != Standards::Language::None);
2294-
if (!force)
2295-
{
2296-
ASSERT_LANG(mLang == Standards::Language::None);
2297-
}
2298-
2299-
mLang = lang;
2300-
}
2301-
23022283
const Token * TokenList::isFunctionHead(const Token *tok, const std::string &endsWith)
23032284
{
23042285
if (!tok)

‎lib/tokenlist.h

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ struct TokensFrontBack {
5151
class CPPCHECKLIB TokenList {
5252
public:
5353
// TODO: pass settings as reference
54-
explicit TokenList(const Settings* settings);
54+
explicit TokenList(const Settings* settings, Standards::Language lang);
5555
~TokenList();
5656

5757
TokenList(const TokenList &) = delete;
@@ -68,9 +68,6 @@ class CPPCHECKLIB TokenList {
6868
/** @return true if the code is C++ */
6969
bool isCPP() const;
7070

71-
// TODO: get rid of this
72-
void setLang(Standards::Language lang, bool force = false);
73-
7471
/**
7572
* Delete all tokens in given token list
7673
* @param tok token list to delete
@@ -103,9 +100,8 @@ class CPPCHECKLIB TokenList {
103100
* - UTF in the code are not handled.
104101
* - comments are not handled.
105102
* @param code input stream for code
106-
* @param lang the language of the code
107103
*/
108-
bool createTokens(std::istream &code, Standards::Language lang);
104+
bool createTokens(std::istream &code);
109105

110106
void createTokens(simplecpp::TokenList&& tokenList);
111107

‎lib/valueflow.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1943,9 +1943,9 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
19431943
}
19441944
static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
19451945
{
1946-
TokenList tokenList(nullptr);
1946+
TokenList tokenList(nullptr, cpp ? Standards::Language::CPP : Standards::Language::C);
19471947
std::istringstream istr(y);
1948-
tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
1948+
tokenList.createTokens(istr); // TODO: check result?
19491949
return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
19501950
}
19511951
static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
@@ -7051,9 +7051,9 @@ static bool getMinMaxValues(const std::string& typestr,
70517051
MathLib::bigint& minvalue,
70527052
MathLib::bigint& maxvalue)
70537053
{
7054-
TokenList typeTokens(&settings);
7054+
TokenList typeTokens(&settings, cpp ? Standards::Language::CPP : Standards::Language::C);
70557055
std::istringstream istr(typestr + ";");
7056-
if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
7056+
if (!typeTokens.createTokens(istr))
70577057
return false;
70587058
typeTokens.simplifyPlatformTypes();
70597059
typeTokens.simplifyStdType();

‎test/helpers.h

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -45,21 +45,16 @@ namespace tinyxml2 {
4545
class SimpleTokenizer : public Tokenizer {
4646
public:
4747
explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
48-
: Tokenizer{TokenList{&s_settings}, s_settings, errorlogger}
49-
{
50-
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
51-
}
48+
: Tokenizer{TokenList{&s_settings, cpp ? Standards::Language::CPP : Standards::Language::C}, s_settings, errorlogger}
49+
{}
5250

5351
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
54-
: Tokenizer{TokenList{&settings}, settings, errorlogger}
55-
{
56-
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
57-
}
52+
: Tokenizer{TokenList{&settings, cpp ? Standards::Language::CPP : Standards::Language::C}, settings, errorlogger}
53+
{}
5854

5955
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
60-
: Tokenizer{TokenList{&settings}, settings, errorlogger}
56+
: Tokenizer{TokenList{&settings, Path::identify(filename, false)}, settings, errorlogger}
6157
{
62-
list.setLang(Path::identify(filename, false));
6358
list.appendFileIfNew(filename);
6459
}
6560

@@ -89,7 +84,7 @@ class SimpleTokenizer : public Tokenizer {
8984
if (list.front())
9085
throw std::runtime_error("token list is not empty");
9186
list.appendFileIfNew(filename);
92-
if (!list.createTokens(istr, Path::identify(filename, false)))
87+
if (!list.createTokens(istr))
9388
return false;
9489

9590
return simplifyTokens1("");
@@ -104,9 +99,10 @@ class SimpleTokenList
10499
public:
105100
template<size_t size>
106101
explicit SimpleTokenList(const char (&code)[size], Standards::Language lang = Standards::Language::CPP)
102+
: list{&settings, lang}
107103
{
108104
std::istringstream iss(code);
109-
if (!list.createTokens(iss, lang))
105+
if (!list.createTokens(iss))
110106
throw std::runtime_error("creating tokens failed");
111107
}
112108

@@ -120,7 +116,7 @@ class SimpleTokenList
120116

121117
private:
122118
const Settings settings;
123-
TokenList list{&settings};
119+
TokenList list;
124120
};
125121

126122

@@ -238,14 +234,14 @@ class SimpleTokenizer2 : public Tokenizer {
238234
public:
239235
template<size_t size>
240236
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
241-
: Tokenizer{TokenList{&settings}, settings, errorlogger}
237+
: Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger}
242238
{
243239
preprocess(code, mFiles, file0, *this, errorlogger);
244240
}
245241

246242
// TODO: get rid of this
247243
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
248-
: Tokenizer{TokenList{&settings}, settings, errorlogger}
244+
: Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger}
249245
{
250246
preprocess(code, mFiles, file0, *this, errorlogger);
251247
}
@@ -263,7 +259,7 @@ struct TokenListHelper
263259
if (tokenlist.front())
264260
throw std::runtime_error("token list is not empty");
265261
tokenlist.appendFileIfNew(file);
266-
return tokenlist.createTokens(istr, Path::identify(file, false));
262+
return tokenlist.createTokens(istr);
267263
}
268264
};
269265

‎test/testclangimport.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ class TestClangImport : public TestFixture {
140140

141141
std::string parse(const char clang[]) {
142142
const Settings settings = settingsBuilder().clang().build();
143-
TokenList tokenlist{&settings};
143+
TokenList tokenlist{&settings, Standards::Language::CPP};
144144
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
145145
std::istringstream istr(clang);
146146
clangimport::parseClangAstDump(tokenizer, istr);
@@ -1061,7 +1061,7 @@ class TestClangImport : public TestFixture {
10611061

10621062
#define GET_SYMBOL_DB(AST) \
10631063
const Settings settings = settingsBuilder().clang().platform(Platform::Type::Unix64).build(); \
1064-
TokenList tokenlist{&settings}; \
1064+
TokenList tokenlist{&settings, Standards::Language::CPP}; \
10651065
Tokenizer tokenizer(std::move(tokenlist), settings, *this); \
10661066
{ \
10671067
std::istringstream istr(AST); \

0 commit comments

Comments
 (0)