Skip to content

Commit 9c9d08e

Browse files
firewavefirewave
authored
Token: avoid unnecessary lookups in update_property_info() and added unit tests for it (#7369)
1 parent 3363525 commit 9c9d08e

File tree

3 files changed

+272
-5
lines changed

3 files changed

+272
-5
lines changed

‎lib/token.cpp

+18-5
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,8 @@ static const std::unordered_set<std::string> controlFlowKeywords = {
107107

108108
void Token::update_property_info()
109109
{
110-
setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
110+
setFlag(fIsControlFlowKeyword, false);
111+
// TODO: clear fIsLong
111112
isStandardType(false);
112113

113114
if (!mStr.empty()) {
@@ -124,11 +125,21 @@ void Token::update_property_info()
124125
else if (std::isalpha((unsigned char)mStr[0]) || mStr[0] == '_' || mStr[0] == '$') { // Name
125126
if (mImpl->mVarId)
126127
tokType(eVariable);
127-
else if (mTokensFrontBack.list.isKeyword(mStr) || mStr == "asm") // TODO: not a keyword
128+
else if (mTokensFrontBack.list.isKeyword(mStr)) {
128129
tokType(eKeyword);
130+
update_property_isStandardType();
131+
if (mTokType != eType) // cannot be a control-flow keyword when it is a type
132+
setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
133+
}
134+
else if (mStr == "asm") { // TODO: not a keyword
135+
tokType(eKeyword);
136+
}
129137
// TODO: remove condition? appears to be (no longer necessary) protection for reset of varids in Tokenizer::setVarId()
130-
else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword)
138+
else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword) {
131139
tokType(eName);
140+
// some types are not being treated as keywords
141+
update_property_isStandardType();
142+
}
132143
} else if (simplecpp::Token::isNumberLike(mStr)) {
133144
if ((MathLib::isInt(mStr) || MathLib::isFloat(mStr)) && mStr.find('_') == std::string::npos)
134145
tokType(eNumber);
@@ -148,6 +159,7 @@ void Token::update_property_info()
148159
mStr == "||" ||
149160
mStr == "!"))
150161
tokType(eLogicalOp);
162+
// TODO: should link check only apply to < and >? Token::link() suggests so
151163
else if (mStr.size() <= 2 && !mLink &&
152164
(mStr == "==" ||
153165
mStr == "!=" ||
@@ -168,11 +180,12 @@ void Token::update_property_info()
168180
tokType(eEllipsis);
169181
else
170182
tokType(eOther);
171-
172-
update_property_isStandardType();
173183
} else {
174184
tokType(eNone);
175185
}
186+
// TODO: make sure varid is only set for eVariable
187+
//assert(!mImpl->mVarId || mTokType == eVariable);
188+
// TODO: validate type for linked token?
176189
}
177190

178191
static const std::unordered_set<std::string> stdTypes = { "bool"

‎lib/token.h

+3
Original file line numberDiff line numberDiff line change
@@ -956,9 +956,12 @@ class CPPCHECKLIB Token {
956956
return;
957957

958958
mImpl->mVarId = id;
959+
// TODO: remove special handling?
959960
if (id != 0) {
960961
tokType(eVariable);
961962
isStandardType(false);
963+
// TODO: clear fIsLong
964+
// TODO: clear fIsControlFlowKeyword
962965
} else {
963966
update_property_info();
964967
}

‎test/testtoken.cpp

+251
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,14 @@ class TestToken : public TestFixture {
115115

116116
TEST_CASE(hasKnownIntValue);
117117

118+
TEST_CASE(update_property_info);
119+
TEST_CASE(update_property_info_evariable);
120+
TEST_CASE(update_property_info_ekeyword_c);
121+
TEST_CASE(update_property_info_ekeyword_cpp);
122+
TEST_CASE(update_property_info_ebracket_link);
123+
TEST_CASE(update_property_info_ecomparisonop_link);
124+
TEST_CASE(update_property_info_etype_c);
125+
TEST_CASE(update_property_info_etype_cpp);
118126
TEST_CASE(varid_reset);
119127
}
120128

@@ -1245,6 +1253,249 @@ class TestToken : public TestFixture {
12451253
_assert_tok(file, line, &tok, t, l, std, ctrl);
12461254
}
12471255

1256+
void update_property_info() const
1257+
{
1258+
assert_tok("", Token::Type::eNone);
1259+
assert_tok("true", Token::Type::eBoolean);
1260+
assert_tok("false", Token::Type::eBoolean);
1261+
assert_tok("\"\"", Token::Type::eString);
1262+
assert_tok("L\"\"", Token::Type::eString, /*l=*/ true);
1263+
assert_tok("'a'", Token::Type::eChar);
1264+
assert_tok("L'a'", Token::Type::eChar, /*l=*/ true);
1265+
// eVariable has separate test
1266+
// eKeyword with specific languages and standards has separate tests
1267+
assert_tok("sizeof", Token::Type::eKeyword);
1268+
assert_tok("goto", Token::Type::eKeyword, /*l=*/ false, /*std=*/ false, /*ctrl=*/ true);
1269+
assert_tok("asm", Token::Type::eKeyword);
1270+
assert_tok("a", Token::Type::eName);
1271+
assert_tok("_a", Token::Type::eName);
1272+
assert_tok("$a", Token::Type::eName);
1273+
assert_tok("bool2", Token::Type::eName);
1274+
assert_tok("0", Token::Type::eNumber);
1275+
assert_tok("-0", Token::Type::eNumber);
1276+
assert_tok("+0", Token::Type::eNumber);
1277+
assert_tok("0xa", Token::Type::eNumber);
1278+
assert_tok("010", Token::Type::eNumber);
1279+
assert_tok("0b0", Token::Type::eNumber);
1280+
assert_tok("0.0", Token::Type::eNumber);
1281+
assert_tok("0x0.3p10", Token::Type::eNumber);
1282+
assert_tok("0z", Token::Type::eNumber); // TODO: not a valid number
1283+
assert_tok("0_km", Token::Type::eName); // user literal
1284+
assert_tok("=", Token::Type::eAssignmentOp);
1285+
assert_tok("<<=", Token::Type::eAssignmentOp);
1286+
assert_tok(">>=", Token::Type::eAssignmentOp);
1287+
assert_tok("+=", Token::Type::eAssignmentOp);
1288+
assert_tok("-=", Token::Type::eAssignmentOp);
1289+
assert_tok("*=", Token::Type::eAssignmentOp);
1290+
assert_tok("/=", Token::Type::eAssignmentOp);
1291+
assert_tok("%=", Token::Type::eAssignmentOp);
1292+
assert_tok("&=", Token::Type::eAssignmentOp);
1293+
assert_tok("|=", Token::Type::eAssignmentOp);
1294+
assert_tok("^=", Token::Type::eAssignmentOp);
1295+
assert_tok(",", Token::Type::eExtendedOp);
1296+
assert_tok("[", Token::Type::eExtendedOp);
1297+
assert_tok("]", Token::Type::eExtendedOp);
1298+
assert_tok("(", Token::Type::eExtendedOp);
1299+
assert_tok(")", Token::Type::eExtendedOp);
1300+
assert_tok("?", Token::Type::eExtendedOp);
1301+
assert_tok(":", Token::Type::eExtendedOp);
1302+
assert_tok("<<", Token::Type::eArithmeticalOp);
1303+
assert_tok(">>", Token::Type::eArithmeticalOp);
1304+
assert_tok("+", Token::Type::eArithmeticalOp);
1305+
assert_tok("-", Token::Type::eArithmeticalOp);
1306+
assert_tok("*", Token::Type::eArithmeticalOp);
1307+
assert_tok("/", Token::Type::eArithmeticalOp);
1308+
assert_tok("%", Token::Type::eArithmeticalOp);
1309+
assert_tok("&", Token::Type::eBitOp);
1310+
assert_tok("|", Token::Type::eBitOp);
1311+
assert_tok("^", Token::Type::eBitOp);
1312+
assert_tok("~", Token::Type::eBitOp);
1313+
assert_tok("&&", Token::Type::eLogicalOp);
1314+
assert_tok("||", Token::Type::eLogicalOp);
1315+
assert_tok("!", Token::Type::eLogicalOp);
1316+
assert_tok("==", Token::Type::eComparisonOp);
1317+
assert_tok("!=", Token::Type::eComparisonOp);
1318+
assert_tok("<", Token::Type::eComparisonOp);
1319+
assert_tok("<=", Token::Type::eComparisonOp);
1320+
assert_tok(">", Token::Type::eComparisonOp);
1321+
assert_tok(">=", Token::Type::eComparisonOp);
1322+
// eComparisonOp with link has a separate test
1323+
assert_tok("<=>", Token::Type::eComparisonOp);
1324+
assert_tok("++", Token::Type::eIncDecOp);
1325+
assert_tok("--", Token::Type::eIncDecOp);
1326+
assert_tok("{", Token::Type::eBracket);
1327+
assert_tok("}", Token::Type::eBracket);
1328+
// < and > with link have a separate test
1329+
assert_tok("...", Token::Type::eEllipsis);
1330+
assert_tok(";", Token::Type::eOther);
1331+
// eType with specific languages and standards has separate tests
1332+
assert_tok("void", Token::Type::eType, /*l=*/ false, /*std=*/ true);
1333+
}
1334+
1335+
void update_property_info_evariable() const
1336+
{
1337+
{
1338+
TokensFrontBack tokensFrontBack(list);
1339+
Token tok(tokensFrontBack);
1340+
tok.str("var1");
1341+
tok.varId(17);
1342+
assert_tok(&tok, Token::Type::eVariable);
1343+
}
1344+
{
1345+
TokensFrontBack tokensFrontBack(list);
1346+
Token tok(tokensFrontBack);
1347+
tok.varId(17);
1348+
tok.str("var1");
1349+
assert_tok(&tok, Token::Type::eVariable);
1350+
}
1351+
}
1352+
1353+
void update_property_info_ekeyword_c() const
1354+
{
1355+
{
1356+
const Settings s = settingsBuilder().c(Standards::cstd_t::C89).build();
1357+
TokenList list_c{&s};
1358+
list_c.setLang(Standards::Language::C);
1359+
TokensFrontBack tokensFrontBack(list_c);
1360+
Token tok(tokensFrontBack);
1361+
tok.str("alignas"); // not a C89 keyword
1362+
assert_tok(&tok, Token::Type::eName);
1363+
}
1364+
{
1365+
TokenList list_c{&settingsDefault};
1366+
list_c.setLang(Standards::Language::C);
1367+
TokensFrontBack tokensFrontBack(list_c);
1368+
Token tok(tokensFrontBack);
1369+
tok.str("alignas"); // a C23 keyword
1370+
assert_tok(&tok, Token::Type::eKeyword);
1371+
}
1372+
{
1373+
TokenList list_c{&settingsDefault};
1374+
list_c.setLang(Standards::Language::C);
1375+
TokensFrontBack tokensFrontBack(list_c);
1376+
Token tok(tokensFrontBack);
1377+
tok.str("and_eq"); // a C++ keyword
1378+
assert_tok(&tok, Token::Type::eName);
1379+
}
1380+
}
1381+
1382+
void update_property_info_ekeyword_cpp() const
1383+
{
1384+
{
1385+
const Settings s = settingsBuilder().cpp(Standards::cppstd_t::CPP03).build();
1386+
TokenList list_cpp{&s};
1387+
list_cpp.setLang(Standards::Language::CPP);
1388+
TokensFrontBack tokensFrontBack(list_cpp);
1389+
Token tok(tokensFrontBack);
1390+
tok.str("consteval"); // not a C++03 keyword
1391+
assert_tok(&tok, Token::Type::eName);
1392+
}
1393+
{
1394+
TokenList list_cpp{&settingsDefault};
1395+
list_cpp.setLang(Standards::Language::CPP);
1396+
TokensFrontBack tokensFrontBack(list_cpp);
1397+
Token tok(tokensFrontBack);
1398+
tok.str("consteval"); // a C++20 keyword
1399+
assert_tok(&tok, Token::Type::eKeyword);
1400+
}
1401+
{
1402+
TokenList list_cpp{&settingsDefault};
1403+
list_cpp.setLang(Standards::Language::CPP);
1404+
TokensFrontBack tokensFrontBack(list_cpp);
1405+
Token tok(tokensFrontBack);
1406+
tok.str("typeof_unqual"); // a C keyword
1407+
assert_tok(&tok, Token::Type::eName);
1408+
}
1409+
}
1410+
1411+
void update_property_info_ebracket_link() const
1412+
{
1413+
{
1414+
TokensFrontBack tokensFrontBack(list);
1415+
Token tok(tokensFrontBack);
1416+
tok.str("<");
1417+
1418+
Token tok2(tokensFrontBack);
1419+
tok.link(&tok2);
1420+
assert_tok(&tok, Token::Type::eBracket);
1421+
}
1422+
1423+
{
1424+
TokensFrontBack tokensFrontBack(list);
1425+
Token tok(tokensFrontBack);
1426+
1427+
Token tok2(tokensFrontBack);
1428+
tok.link(&tok2);
1429+
1430+
tok.str("<");
1431+
assert_tok(&tok, Token::Type::eBracket);
1432+
}
1433+
}
1434+
1435+
void update_property_info_ecomparisonop_link() const
1436+
{
1437+
{
1438+
TokensFrontBack tokensFrontBack(list);
1439+
Token tok(tokensFrontBack);
1440+
tok.str("==");
1441+
1442+
Token tok2(tokensFrontBack);
1443+
tok.link(&tok2); // TODO: does not (and probably should not) update
1444+
assert_tok(&tok, Token::Type::eComparisonOp);
1445+
}
1446+
1447+
{
1448+
TokensFrontBack tokensFrontBack(list);
1449+
Token tok(tokensFrontBack);
1450+
1451+
Token tok2(tokensFrontBack);
1452+
tok.link(&tok2);
1453+
1454+
tok.str("==");
1455+
assert_tok(&tok, Token::Type::eOther); // TODO: this looks wrong
1456+
}
1457+
}
1458+
1459+
void update_property_info_etype_c() const
1460+
{
1461+
{
1462+
TokenList list_c{&settingsDefault};
1463+
list_c.setLang(Standards::Language::C);
1464+
TokensFrontBack tokensFrontBack(list_c);
1465+
Token tok(tokensFrontBack);
1466+
tok.str("char"); // not treated as keyword in TokenList::isKeyword()
1467+
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
1468+
}
1469+
{
1470+
TokenList list_c{&settingsDefault};
1471+
list_c.setLang(Standards::Language::C);
1472+
TokensFrontBack tokensFrontBack(list_c);
1473+
Token tok(tokensFrontBack);
1474+
tok.str("size_t"); // not treated as keyword in TokenList::isKeyword()
1475+
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
1476+
}
1477+
}
1478+
1479+
void update_property_info_etype_cpp() const
1480+
{
1481+
{
1482+
TokenList list_cpp{&settingsDefault};
1483+
list_cpp.setLang(Standards::Language::CPP);
1484+
TokensFrontBack tokensFrontBack(list_cpp);
1485+
Token tok(tokensFrontBack);
1486+
tok.str("bool"); // not treated as keyword in TokenList::isKeyword()
1487+
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
1488+
}
1489+
{
1490+
TokenList list_cpp{&settingsDefault};
1491+
list_cpp.setLang(Standards::Language::CPP);
1492+
TokensFrontBack tokensFrontBack(list_cpp);
1493+
Token tok(tokensFrontBack);
1494+
tok.str("size_t");
1495+
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
1496+
}
1497+
}
1498+
12481499
void varid_reset() const
12491500
{
12501501
TokenList list_c{&settingsDefault};

0 commit comments

Comments
 (0)