From eebfbada0af5e725acceb81b4e5daca227f9311f Mon Sep 17 00:00:00 2001 From: glank Date: Mon, 20 Jul 2026 15:46:34 +0200 Subject: [PATCH 1/7] Keep line directives in tokenized output --- simplecpp.cpp | 27 +++------------------------ simplecpp.h | 1 - 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index d2c398c4..70290700 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -640,23 +640,6 @@ static bool isStringLiteralPrefix(const std::string &str) str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R"; } -void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location) -{ - if (fileIndex_ != location.fileIndex || line >= location.line) { - location.fileIndex = fileIndex_; - location.line = line; - return; - } - - if (line + 2 >= location.line) { - location.line = line; - while (cback()->op != '#') - deleteToken(back()); - deleteToken(back()); - return; - } -} - static const std::string COMMENT_END("*/"); void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, OutputList *outputList) @@ -726,17 +709,13 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (!ppTok || !ppTok->number) continue; - const unsigned int line = std::atol(ppTok->str().c_str()); - ppTok = advanceAndSkipComments(ppTok); + location.line = std::atol(ppTok->str().c_str()); - unsigned int fileindex; + ppTok = advanceAndSkipComments(ppTok); if (ppTok && ppTok->str()[0] == '\"') - fileindex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\")); - else - fileindex = location.fileIndex; + location.fileIndex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\")); - lineDirective(fileindex, line, location); } continue; diff --git a/simplecpp.h b/simplecpp.h index 3d6fc526..66f084fd 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -396,7 +396,6 @@ namespace simplecpp { void constFoldQuestionOp(Token *&tok1); std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList); - void lineDirective(unsigned int fileIndex_, unsigned int line, Location &location); const Token* lastLineTok(int maxsize=1000) const; const Token* isLastLinePreprocessor(int maxsize=1000) const; From 2448c369f2db09b0c2afc60e3956c7b83ac19018 Mon Sep 17 00:00:00 2001 From: glank Date: Mon, 20 Jul 2026 17:24:56 +0200 Subject: [PATCH 2/7] Fix gotoNextLine --- simplecpp.cpp | 14 ++++++++++++-- simplecpp.h | 5 +++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 70290700..6377d396 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -648,6 +648,8 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, const Token *oldLastToken = nullptr; + bool locationchange = false; + Location location(fileIndex(filename), 1, 1); while (stream.good()) { unsigned char ch = stream.readChar(); @@ -716,6 +718,7 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (ppTok && ppTok->str()[0] == '\"') location.fileIndex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\")); + locationchange = true; } continue; @@ -934,6 +937,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, push_back(new Token(currentToken, location, !!std::isspace(stream.peekChar()))); + if (locationchange) { + back()->locationchange = true; + locationchange = false; + } + if (multiline) location.col += currentToken.size(); else @@ -1435,7 +1443,7 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const const Token* prevTok = nullptr; int count = 0; for (const Token *tok = cback(); ; tok = tok->previous) { - if (!sameline(tok, cback())) + if (!sameline(tok, cback()) || tok->locationchange) break; if (tok->comment) continue; @@ -2979,7 +2987,9 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok) { const unsigned int line = tok->location.line; const unsigned int file = tok->location.fileIndex; - while (tok && tok->location.line == line && tok->location.fileIndex == file) + if (tok) + tok = tok->next; + while (tok && tok->location.line == line && tok->location.fileIndex == file && !tok->locationchange) tok = tok->next; return tok; } diff --git a/simplecpp.h b/simplecpp.h index 66f084fd..750478be 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -151,12 +151,12 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc, bool wsahead = false) : - whitespaceahead(wsahead), location(loc), string(s) { + whitespaceahead(wsahead), locationchange(false), location(loc), string(s) { flags(); } Token(const Token &tok) : - macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} + macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), locationchange(tok.locationchange), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {} Token &operator=(const Token &tok) = delete; @@ -182,6 +182,7 @@ namespace simplecpp { bool name; bool number; bool whitespaceahead; + bool locationchange; /* token location is at a discontinuity from #line directive */ Location location; Token *previous{}; Token *next{}; From 31bf2b81c8501118d9fc6fcf1674b592e0447e8d Mon Sep 17 00:00:00 2001 From: glank Date: Mon, 20 Jul 2026 17:51:58 +0200 Subject: [PATCH 3/7] Fix lastLineTok --- simplecpp.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 6377d396..110cab5b 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -1443,8 +1443,10 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const const Token* prevTok = nullptr; int count = 0; for (const Token *tok = cback(); ; tok = tok->previous) { - if (!sameline(tok, cback()) || tok->locationchange) + if (!sameline(tok, cback())) break; + if (tok->locationchange) + return tok; if (tok->comment) continue; if (++count > maxsize) From 1821aadca66ce4dbbcdc1187ffc171f8e99b1b40 Mon Sep 17 00:00:00 2001 From: glank Date: Mon, 20 Jul 2026 17:56:51 +0200 Subject: [PATCH 4/7] Fix clang-tidy issues --- simplecpp.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/simplecpp.h b/simplecpp.h index 750478be..25a3e3fa 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -151,7 +151,7 @@ namespace simplecpp { class SIMPLECPP_LIB Token { public: Token(const TokenString &s, const Location &loc, bool wsahead = false) : - whitespaceahead(wsahead), locationchange(false), location(loc), string(s) { + whitespaceahead(wsahead), location(loc), string(s) { flags(); } @@ -182,7 +182,8 @@ namespace simplecpp { bool name; bool number; bool whitespaceahead; - bool locationchange; /* token location is at a discontinuity from #line directive */ + /** whether token location is at a discontinuity from a #line directive */ + bool locationchange {false}; Location location; Token *previous{}; Token *next{}; From 4c6ba367e0488497c68f2740a743b9f601818168 Mon Sep 17 00:00:00 2001 From: glank Date: Tue, 21 Jul 2026 11:35:08 +0200 Subject: [PATCH 5/7] Add tests --- simplecpp.cpp | 2 +- test.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 110cab5b..b48afaf4 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3532,7 +3532,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL continue; } - if (rawtok->op == '#' && !sameline(rawtok->previousSkipComments(), rawtok)) { + if (rawtok->op == '#' && (!sameline(rawtok->previousSkipComments(), rawtok) || rawtok->locationchange)) { if (!sameline(rawtok, rawtok->next)) { rawtok = rawtok->next; continue; diff --git a/test.cpp b/test.cpp index b13140aa..12fc2793 100644 --- a/test.cpp +++ b/test.cpp @@ -2481,9 +2481,9 @@ static void location12() "/**//**/#/**//**/line/**//**/3/**//**/\"file.c\"/**/\n" "__LINE__ __FILE__\n"; ASSERT_EQUALS("\n" - "#line 3 \"file.c\"\n" - "3 \"file.c\"", - preprocess(code)); + "#line 3 \"file.c\"\n" + "3 \"file.c\"", + preprocess(code)); } static void missingHeader1() @@ -2749,6 +2749,62 @@ static void nullDirective3() ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code)); } +static void lineDirective1() +{ + const char code[] = "#line 1\n" + "#line 2 \"header1.h\"\n" + "# 1\n" + "# 2 \"header2.h\"\n" + "# 3 \"header2.h\" 0\n"; + + std::vector files; + const simplecpp::TokenList rawtokens(code, files); + + ASSERT_EQUALS("1: # line 1 # line 2 \"header1.h\"\n" + "#line 2 \"header1.h\"\n" + "2: # 1\n" + "#line 1 \"header1.h\"\n" + "1: # 2 \"header2.h\"\n" + "#line 2 \"header2.h\"\n" + "2: # 3 \"header2.h\" 0", + rawtokens.stringify(true)); + + simplecpp::FileDataCache cache; + simplecpp::TokenList out(files); + simplecpp::DUI dui; + simplecpp::preprocess(out, rawtokens, files, cache, dui); + + ASSERT_EQUALS("", out.stringify(true)); +} + +static void lineDirective2() +{ + const char code[] = "#\n" + "#line 1\n" + "include\n" + "#line 1\n" + "\"header1.h\"\n"; + + std::vector files; + const simplecpp::TokenList rawtokens(code, files); + + ASSERT_EQUALS("1: #\n" + "2: # line 1\n" + "#line 1 \"\"\n" + "1: include\n" + "2: # line 1\n" + "#line 1 \"\"\n" + "1: \"header1.h\"", + rawtokens.stringify(true)); + + simplecpp::FileDataCache cache; + simplecpp::TokenList out(files); + simplecpp::DUI dui; + simplecpp::preprocess(out, rawtokens, files, cache, dui); + + ASSERT_EQUALS("1: include \"header1.h\"", out.stringify(true)); +} + static void include1() { const char code[] = "#include \"A.h\"\n"; @@ -4167,6 +4223,9 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(nullDirective2); TEST_CASE(nullDirective3); + TEST_CASE(lineDirective1); + TEST_CASE(lineDirective2); + TEST_CASE(include1); TEST_CASE(include2); TEST_CASE(include3); From 8a226f30876ce3f6a6b3df4c632e8cddc1cc03ca Mon Sep 17 00:00:00 2001 From: glank Date: Tue, 21 Jul 2026 11:38:41 +0200 Subject: [PATCH 6/7] Fix clang-tidy issues --- test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index 12fc2793..7cbe74af 100644 --- a/test.cpp +++ b/test.cpp @@ -2771,7 +2771,7 @@ static void lineDirective1() simplecpp::FileDataCache cache; simplecpp::TokenList out(files); - simplecpp::DUI dui; + const simplecpp::DUI dui; simplecpp::preprocess(out, rawtokens, files, cache, dui); ASSERT_EQUALS("", out.stringify(true)); @@ -2799,7 +2799,7 @@ static void lineDirective2() simplecpp::FileDataCache cache; simplecpp::TokenList out(files); - simplecpp::DUI dui; + const simplecpp::DUI dui; simplecpp::preprocess(out, rawtokens, files, cache, dui); ASSERT_EQUALS("1: include \"header1.h\"", out.stringify(true)); From 6cbce179e84c59292bc1eecbb7c14b8c8f01fb57 Mon Sep 17 00:00:00 2001 From: glank Date: Tue, 21 Jul 2026 11:40:06 +0200 Subject: [PATCH 7/7] Fix formatting --- test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index 7cbe74af..e141ea07 100644 --- a/test.cpp +++ b/test.cpp @@ -2481,9 +2481,9 @@ static void location12() "/**//**/#/**//**/line/**//**/3/**//**/\"file.c\"/**/\n" "__LINE__ __FILE__\n"; ASSERT_EQUALS("\n" - "#line 3 \"file.c\"\n" - "3 \"file.c\"", - preprocess(code)); + "#line 3 \"file.c\"\n" + "3 \"file.c\"", + preprocess(code)); } static void missingHeader1()