Skip to content

Commit fadc93f

Browse files
firewavefirewave
authored
testrunner: refactored givenACodeSampleToTokenize into SimpleTokenizer and SimpleTokenList / several related cleanups (#6084)
1 parent f9f3fe0 commit fadc93f

File tree

9 files changed

+289
-287
lines changed

9 files changed

+289
-287
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -856,10 +856,10 @@ test/testtoken.o: test/testtoken.cpp lib/addoninfo.h lib/check.h lib/color.h lib
856856
test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
857857
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenize.cpp
858858

859-
test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
859+
test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
860860
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenlist.cpp
861861

862-
test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h
862+
test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
863863
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenrange.cpp
864864

865865
test/testtype.o: test/testtype.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/checktype.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h

‎test/helpers.h‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,34 +20,32 @@
2020
#define helpersH
2121

2222
#include "settings.h"
23+
#include "standards.h"
2324
#include "tokenize.h"
2425
#include "tokenlist.h"
2526

2627
#include <cstddef>
28+
#include <stdexcept>
2729
#include <sstream>
2830
#include <string>
2931
#include <vector>
3032

3133
class Token;
3234
class Preprocessor;
3335
class SuppressionList;
36+
class ErrorLogger;
3437
namespace simplecpp {
3538
struct DUI;
3639
}
3740

38-
class givenACodeSampleToTokenize {
39-
private:
40-
const Settings settings;
41-
Tokenizer tokenizer;
42-
41+
class SimpleTokenizer {
4342
public:
44-
explicit givenACodeSampleToTokenize(const char sample[], bool createOnly = false, bool cpp = true)
45-
: tokenizer(settings, nullptr) {
43+
SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
44+
: tokenizer{settings, &errorlogger}
45+
{
4646
std::istringstream iss(sample);
47-
if (createOnly)
48-
tokenizer.list.createTokens(iss, cpp ? "test.cpp" : "test.c");
49-
else
50-
tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c");
47+
if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
48+
throw std::runtime_error("creating tokens failed");
5149
}
5250

5351
Token* tokens() {
@@ -57,6 +55,34 @@ class givenACodeSampleToTokenize {
5755
const Token* tokens() const {
5856
return tokenizer.tokens();
5957
}
58+
59+
private:
60+
const Settings settings;
61+
Tokenizer tokenizer;
62+
};
63+
64+
class SimpleTokenList
65+
{
66+
public:
67+
68+
explicit SimpleTokenList(const char code[], Standards::Language lang = Standards::Language::CPP)
69+
{
70+
std::istringstream iss(code);
71+
if (!list.createTokens(iss, lang))
72+
throw std::runtime_error("creating tokens failed");
73+
}
74+
75+
Token* front() {
76+
return list.front();
77+
}
78+
79+
const Token* front() const {
80+
return list.front();
81+
}
82+
83+
private:
84+
const Settings settings;
85+
TokenList list{&settings};
6086
};
6187

6288

‎test/testlibrary.cpp‎

Lines changed: 36 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,8 @@ class TestLibrary : public TestFixture {
117117
" </function>\n"
118118
"</def>";
119119

120-
TokenList tokenList(&settings);
121-
std::istringstream istr("foo();");
122-
tokenList.createTokens(istr, Standards::Language::CPP);
120+
const char code[] = "foo();";
121+
SimpleTokenList tokenList(code);
123122
tokenList.front()->next()->astOperand1(tokenList.front());
124123

125124
Library library;
@@ -140,16 +139,14 @@ class TestLibrary : public TestFixture {
140139
Library library;
141140
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
142141
{
143-
TokenList tokenList(&settings);
144-
std::istringstream istr("fred.foo(123);"); // <- wrong scope, not library function
145-
tokenList.createTokens(istr, Standards::Language::CPP);
142+
const char code[] = "fred.foo(123);"; // <- wrong scope, not library function
143+
const SimpleTokenList tokenList(code);
146144

147145
ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
148146
}
149147
{
150-
TokenList tokenList(&settings);
151-
std::istringstream istr("Fred::foo(123);"); // <- wrong scope, not library function
152-
tokenList.createTokens(istr, Standards::Language::CPP);
148+
const char code[] = "Fred::foo(123);"; // <- wrong scope, not library function
149+
const SimpleTokenList tokenList(code);
153150

154151
ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
155152
}
@@ -165,7 +162,7 @@ class TestLibrary : public TestFixture {
165162

166163
TokenList tokenList(&settings);
167164
std::istringstream istr("foo();"); // <- too few arguments, not library function
168-
tokenList.createTokens(istr, Standards::Language::CPP);
165+
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
169166
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
170167
tokenList.createAst();
171168

@@ -189,7 +186,7 @@ class TestLibrary : public TestFixture {
189186
{
190187
TokenList tokenList(&settings);
191188
std::istringstream istr("foo();"); // <- too few arguments, not library function
192-
tokenList.createTokens(istr, Standards::Language::CPP);
189+
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
193190
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
194191
tokenList.createAst();
195192

@@ -198,7 +195,7 @@ class TestLibrary : public TestFixture {
198195
{
199196
TokenList tokenList(&settings);
200197
std::istringstream istr("foo(a);"); // <- library function
201-
tokenList.createTokens(istr, Standards::Language::CPP);
198+
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
202199
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
203200
tokenList.createAst();
204201

@@ -207,7 +204,7 @@ class TestLibrary : public TestFixture {
207204
{
208205
TokenList tokenList(&settings);
209206
std::istringstream istr("foo(a, b);"); // <- library function
210-
tokenList.createTokens(istr, Standards::Language::CPP);
207+
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
211208
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
212209
tokenList.createAst();
213210

@@ -216,7 +213,7 @@ class TestLibrary : public TestFixture {
216213
{
217214
TokenList tokenList(&settings);
218215
std::istringstream istr("foo(a, b, c);"); // <- too much arguments, not library function
219-
tokenList.createTokens(istr, Standards::Language::CPP);
216+
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
220217
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
221218
tokenList.createAst();
222219

@@ -232,9 +229,8 @@ class TestLibrary : public TestFixture {
232229
" </function>\n"
233230
"</def>";
234231

235-
TokenList tokenList(&settings);
236-
std::istringstream istr("Fred foo(123);"); // <- Variable declaration, not library function
237-
tokenList.createTokens(istr, Standards::Language::CPP);
232+
const char code[] = "Fred foo(123);"; // <- Variable declaration, not library function
233+
SimpleTokenList tokenList(code);
238234
tokenList.front()->next()->astOperand1(tokenList.front());
239235
tokenList.front()->next()->varId(1);
240236

@@ -292,9 +288,8 @@ class TestLibrary : public TestFixture {
292288
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
293289
ASSERT_EQUALS(0, library.functions["foo"].argumentChecks[-1].notuninit);
294290

295-
TokenList tokenList(&settings);
296-
std::istringstream istr("foo(a,b,c,d,e);");
297-
tokenList.createTokens(istr, Standards::Language::CPP);
291+
const char code[] = "foo(a,b,c,d,e);";
292+
SimpleTokenList tokenList(code);
298293
tokenList.front()->next()->astOperand1(tokenList.front());
299294

300295
ASSERT_EQUALS(false, library.isuninitargbad(tokenList.front(), 1));
@@ -317,9 +312,8 @@ class TestLibrary : public TestFixture {
317312
Library library;
318313
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
319314

320-
TokenList tokenList(&settings);
321-
std::istringstream istr("foo(a,b,c,d);");
322-
tokenList.createTokens(istr, Standards::Language::CPP);
315+
const char code[] = "foo(a,b,c,d);";
316+
SimpleTokenList tokenList(code);
323317
tokenList.front()->next()->astOperand1(tokenList.front());
324318

325319
ASSERT(Library::ArgumentChecks::Direction::DIR_IN == library.getArgDirection(tokenList.front(), 1));
@@ -349,9 +343,8 @@ class TestLibrary : public TestFixture {
349343
Library library;
350344
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
351345

352-
TokenList tokenList(&settings);
353-
std::istringstream istr("foo(a,b,c,d,e,f,g,h,i,j,k);");
354-
tokenList.createTokens(istr, Standards::Language::CPP);
346+
const char code[] = "foo(a,b,c,d,e,f,g,h,i,j,k);";
347+
SimpleTokenList tokenList(code);
355348
tokenList.front()->next()->astOperand1(tokenList.front());
356349

357350
// 1-
@@ -491,9 +484,8 @@ class TestLibrary : public TestFixture {
491484
Library library;
492485
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
493486

494-
TokenList tokenList(&settings);
495-
std::istringstream istr("foo(a,b,c,d,e);");
496-
tokenList.createTokens(istr, Standards::Language::CPP);
487+
const char code[] = "foo(a,b,c,d,e);";
488+
SimpleTokenList tokenList(code);
497489
tokenList.front()->next()->astOperand1(tokenList.front());
498490

499491
// arg1: type=strlen arg2
@@ -554,16 +546,14 @@ class TestLibrary : public TestFixture {
554546
ASSERT(library.functions.at("bar").argumentChecks.empty());
555547

556548
{
557-
TokenList tokenList(&settings);
558-
std::istringstream istr("Foo::foo();");
559-
tokenList.createTokens(istr, Standards::Language::CPP);
549+
const char code[] = "Foo::foo();";
550+
const SimpleTokenList tokenList(code);
560551
ASSERT(library.isnotnoreturn(tokenList.front()->tokAt(2)));
561552
}
562553

563554
{
564-
TokenList tokenList(&settings);
565-
std::istringstream istr("bar();");
566-
tokenList.createTokens(istr, Standards::Language::CPP);
555+
const char code[] = "bar();";
556+
const SimpleTokenList tokenList(code);
567557
ASSERT(library.isnotnoreturn(tokenList.front()));
568558
}
569559
}
@@ -635,9 +625,8 @@ class TestLibrary : public TestFixture {
635625
Library library;
636626
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
637627

638-
TokenList tokenList(&settings);
639-
std::istringstream istr("a(); b();");
640-
tokenList.createTokens(istr, Standards::Language::CPP);
628+
const char code[] = "a(); b();";
629+
const SimpleTokenList tokenList(code);
641630

642631
const Library::WarnInfo* a = library.getWarnInfo(tokenList.front());
643632
const Library::WarnInfo* b = library.getWarnInfo(tokenList.front()->tokAt(4));
@@ -791,7 +780,7 @@ class TestLibrary : public TestFixture {
791780
}
792781
}
793782

794-
void container() const {
783+
void container() {
795784
constexpr char xmldata[] = "<?xml version=\"1.0\"?>\n"
796785
"<def>\n"
797786
" <container id=\"A\" startPattern=\"std :: A &lt;\" endPattern=\"&gt; !!::\" itEndPattern=\"&gt; :: iterator\">\n"
@@ -871,7 +860,7 @@ class TestLibrary : public TestFixture {
871860
ASSERT_EQUALS(C.arrayLike_indexOp, true);
872861

873862
{
874-
givenACodeSampleToTokenize var("std::A<int> a;");
863+
const SimpleTokenizer var(*this, "std::A<int> a;");
875864
ASSERT_EQUALS(&A, library.detectContainer(var.tokens()));
876865
ASSERT(!library.detectIterator(var.tokens()));
877866
bool isIterator;
@@ -880,14 +869,14 @@ class TestLibrary : public TestFixture {
880869
}
881870

882871
{
883-
givenACodeSampleToTokenize var("std::A<int>::size_type a_s;");
872+
const SimpleTokenizer var(*this, "std::A<int>::size_type a_s;");
884873
ASSERT(!library.detectContainer(var.tokens()));
885874
ASSERT(!library.detectIterator(var.tokens()));
886875
ASSERT(!library.detectContainerOrIterator(var.tokens()));
887876
}
888877

889878
{
890-
givenACodeSampleToTokenize var("std::A<int>::iterator a_it;");
879+
const SimpleTokenizer var(*this, "std::A<int>::iterator a_it;");
891880
ASSERT(!library.detectContainer(var.tokens()));
892881
ASSERT_EQUALS(&A, library.detectIterator(var.tokens()));
893882
bool isIterator;
@@ -896,7 +885,7 @@ class TestLibrary : public TestFixture {
896885
}
897886

898887
{
899-
givenACodeSampleToTokenize var("std::B<int> b;");
888+
const SimpleTokenizer var(*this, "std::B<int> b;");
900889
ASSERT_EQUALS(&B, library.detectContainer(var.tokens()));
901890
ASSERT(!library.detectIterator(var.tokens()));
902891
bool isIterator;
@@ -905,14 +894,14 @@ class TestLibrary : public TestFixture {
905894
}
906895

907896
{
908-
givenACodeSampleToTokenize var("std::B<int>::size_type b_s;");
897+
const SimpleTokenizer var(*this, "std::B<int>::size_type b_s;");
909898
ASSERT(!library.detectContainer(var.tokens()));
910899
ASSERT(!library.detectIterator(var.tokens()));
911900
ASSERT(!library.detectContainerOrIterator(var.tokens()));
912901
}
913902

914903
{
915-
givenACodeSampleToTokenize var("std::B<int>::iterator b_it;");
904+
const SimpleTokenizer var(*this, "std::B<int>::iterator b_it;");
916905
ASSERT(!library.detectContainer(var.tokens()));
917906
ASSERT_EQUALS(&B, library.detectIterator(var.tokens()));
918907
bool isIterator;
@@ -921,14 +910,14 @@ class TestLibrary : public TestFixture {
921910
}
922911

923912
{
924-
givenACodeSampleToTokenize var("C c;");
913+
const SimpleTokenizer var(*this, "C c;");
925914
ASSERT(!library.detectContainer(var.tokens()));
926915
ASSERT(!library.detectIterator(var.tokens()));
927916
ASSERT(!library.detectContainerOrIterator(var.tokens()));
928917
}
929918

930919
{
931-
givenACodeSampleToTokenize var("D d;");
920+
const SimpleTokenizer var(*this, "D d;");
932921
ASSERT(!library.detectContainer(var.tokens()));
933922
ASSERT(!library.detectIterator(var.tokens()));
934923
ASSERT(!library.detectContainerOrIterator(var.tokens()));

‎test/testsimplifytemplate.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5290,7 +5290,8 @@ class TestSimplifyTemplate : public TestFixture {
52905290
Tokenizer tokenizer(settings, this);
52915291

52925292
std::istringstream istr(code);
5293-
tokenizer.list.createTokens(istr, "test.cpp");
5293+
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5294+
return false;
52945295
tokenizer.createLinks();
52955296
tokenizer.splitTemplateRightAngleBrackets(false);
52965297

@@ -5357,7 +5358,8 @@ class TestSimplifyTemplate : public TestFixture {
53575358
Tokenizer tokenizer(settings, this);
53585359

53595360
std::istringstream istr(code);
5360-
tokenizer.list.createTokens(istr, "test.cpp");
5361+
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5362+
return false;
53615363
tokenizer.createLinks();
53625364
tokenizer.splitTemplateRightAngleBrackets(false);
53635365

@@ -5427,7 +5429,8 @@ class TestSimplifyTemplate : public TestFixture {
54275429
Tokenizer tokenizer(settings, this);
54285430

54295431
std::istringstream istr(code);
5430-
tokenizer.list.createTokens(istr, "test.cpp");
5432+
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5433+
return false;
54315434
tokenizer.createLinks();
54325435
tokenizer.splitTemplateRightAngleBrackets(false);
54335436

@@ -5456,7 +5459,8 @@ class TestSimplifyTemplate : public TestFixture {
54565459
Tokenizer tokenizer(settings, this);
54575460

54585461
std::istringstream istr(code);
5459-
tokenizer.list.createTokens(istr, "test.cpp");
5462+
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5463+
return false;
54605464
tokenizer.createLinks();
54615465
tokenizer.splitTemplateRightAngleBrackets(false);
54625466

0 commit comments

Comments
 (0)