From 9ff0bada4ecd0cc0c6dc836973a5f118d4b8c164 Mon Sep 17 00:00:00 2001 From: eelke Date: Sun, 18 Oct 2026 15:58:13 +0200 Subject: [PATCH] Refactor ConvertLangToSqlString and fix couple of bugs. Verify with unit tests. --- pglablib/util.cpp | 376 ++++++++---------- ...onvert-lang-literals-5c1e8a2f9d4b7360.yaml | 6 + .../pglabtests/tst_ConvertLangToSqlString.cpp | 90 +++++ 3 files changed, 254 insertions(+), 218 deletions(-) create mode 100644 releasenotes/notes/fix-convert-lang-literals-5c1e8a2f9d4b7360.yaml diff --git a/pglablib/util.cpp b/pglablib/util.cpp index e1e5706..25b3934 100644 --- a/pglablib/util.cpp +++ b/pglablib/util.cpp @@ -221,43 +221,160 @@ QString ConvertToMultiLineRawCppString(const QString &in) namespace { - class Token { - public: - enum Type { - Code, - StringLiteral, - SingleLineComment, - MultiLineComment - }; +// index points at the first char of the token. On success index is placed after the opening quote and quote is set to '"' or '\''. +bool readOpeningQuote(const QString &in, int &index, QChar "e) +{ + QChar c = in[index]; + if (c == '\'' || c == '\"') { + quote = c; + ++index; + return true; + } + if (c == 'L' || c == 'U') { + if (in.length() > index + 1 && in[index + 1] == '\"') { + quote = '\"'; + index += 2; + return true; + } + return false; + } + if (c == 'u') { + if (in.length() > index + 2 && in[index + 1] == '8' && in[index + 2] == '\"') { + quote = '\"'; + index += 3; + return true; + } + if (in.length() > index + 1 && in[index + 1] == '\"') { + quote = '\"'; + index += 2; + return true; + } + } + return false; +} - const Type type; - /// Depends on type - /// Code: literal copy of the matched code - /// StringLiteral: the contents of the string literal, escapes have been unescaped - /// *Comment, the text in the comment - const QString data; +int hexDigitValue(QChar c) +{ + if (c >= '0' && c <= '9') return c.unicode() - '0'; + if (c >= 'a' && c <= 'f') return c.unicode() - 'a' + 10; + if (c >= 'A' && c <= 'F') return c.unicode() - 'A' + 10; + return -1; +} - Token(Type type, QString data) - : type(type) - , data(data) - {} - }; +// Reads exactly `count` hex digits starting at `first`. Fails on out-of-range or non-hex input. +bool parseHexDigits(const QString &in, int first, int count, char32_t &value) +{ + if (in.length() < first + count) return false; + value = 0; + for (int i = first; i < first + count; ++i) { + int digit = hexDigitValue(in[i]); + if (digit < 0) return false; + value = value * 16 + static_cast(digit); + } + return true; +} - /// Tokenizer to get SQL out of a piece of programming language code - /// - /// It works by ignoring most input and only get's triggered by string literals - /// and comments. It does return tokens for the code in between just so - class ProgLangTokenizer { - public: - ProgLangTokenizer(const QString &in) - : input(in) - {} +// Appends a code point as UTF-16. Rejects > 0x10FFFF and surrogates D800-DFFF. +bool appendCodePoint(QString &out, char32_t cp) +{ + if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return false; + if (QChar::requiresSurrogates(cp)) { + out += QChar(QChar::highSurrogate(cp)); + out += QChar(QChar::lowSurrogate(cp)); + } + else { + out += QChar(static_cast(cp)); + } + return true; +} - private: - const QString input; - int position = 0; +// index points at the first octal digit. On success index points at the last consumed digit. +bool decodeOctalEscape(const QString &in, int &index, QString &out) +{ + if (in.length() <= index + 2) return false; + int value = 0; + for (int i = index; i <= index + 2; ++i) { + QChar digit = in[i]; + if (digit < '0' || digit > '7') return false; + value = value * 8 + (digit.unicode() - '0'); + } + if (value >= 0x80) return false; + out += static_cast(static_cast(value)); + index += 2; + return true; +} - }; +// index points at the char after the backslash. On success index points at the last consumed char. +bool decodeEscape(const QString &in, int &index, QString &out) +{ + switch (in[index].unicode()) { + case 'a': + out += '\x07'; + break; + case 'b': + out += '\x08'; + break; + case 'f': + out += '\x0c'; + break; + case 'n': + out += '\n'; + break; + case 'r': + case 'v': + break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + return decodeOctalEscape(in, index, out); + case 'x': { + char32_t value = 0; + if (!parseHexDigits(in, index + 1, 2, value)) return false; + out += static_cast(static_cast(value)); + index += 2; + break; + } + case 'u': { + char32_t value = 0; + if (!parseHexDigits(in, index + 1, 4, value)) return false; + out += QChar(static_cast(value)); + index += 4; + break; + } + case 'U': { + char32_t value = 0; + if (!parseHexDigits(in, index + 1, 8, value)) return false; + if (!appendCodePoint(out, value)) return false; + index += 8; + break; + } + default: + out += in[index]; + break; + } + return true; +} + +// index points at the first char after the opening quote. Stops at the closing quote (consumed), at newline (consumed) or at end of input. +bool readStringBody(const QString &in, int &index, QChar quote, QString &out) +{ + while (index < in.length()) { + QChar c = in[index]; + if (c == quote || c == '\n') { + ++index; + return true; + } + if (c == '\\') { + ++index; + if (index == in.length()) return true; + if (!decodeEscape(in, index, out)) return false; + } + else { + out += c; + } + ++index; + } + return true; +} } @@ -275,195 +392,18 @@ QString ConvertLangToSqlString(const QString &in) // until we reach matching end quote // skip whitespace and + or . - QString output; - - enum { - WHITESPACE, - PREFIX, - IN_STRING, - SingleLineComment, - END, - ERROR - } state = WHITESPACE; int index = 0; QChar quote = '\0'; - while (state != ERROR && state != END && index < in.length()) { - if (state == WHITESPACE) { - // skip all whitespace untill we encounter something else - // we also skip concatenation operators. Note this code is not trying to validate - // for correct syntax so it will quite happily accept many incorrect constructs - // that doesn't matter however as we are just trying to strip everything which is not SQL. - while (index < in.length() && (in[index].isSpace() || in[index] == '+' || in[index] == '.')) ++index; - if (index == in.length()) { - state = END; - break; - } - // Assume quotes can vary - if (in[index] == '\'' || in[index] == '\"') { - quote = in[index]; - ++index; - state = IN_STRING; - } - else { - state = PREFIX; - } - } - else if (state == PREFIX) { - auto c = in[index]; - if (c == 'L' || c == 'U') { - // C++ prefix expect C++ double quote - if (in.length() > index+ 1 && in[index+1] == '"') { - index += 2; - state = IN_STRING; - } - else { - state = ERROR; - break; - } - } - if (c == 'u') { - // C++ prefix expect C++ double quote - if (in.length() > index+ 2 && in[index+1] == '8' && in[index+2] == '"') { - index += 3; - state = IN_STRING; - } - else if (in.length() > index+ 1 && in[index+1] == '"') { - index += 2; - state = IN_STRING; - } - else { - state = ERROR; - break; - } - } - else { - state = ERROR; - break; - } - } - else if (state == IN_STRING) { - // scan contents of string and process any escapes encountered - bool escape = false; - while (state != ERROR && index < in.length()) { - QChar c = in[index]; - if (escape) { - if (c == 'a') output += '\x07'; - else if (c == 'b') output += '\x08'; - else if (c == 'f') output += '\x0c'; - else if (c == 'n') output += '\n'; - else if (c == 'r') ; - else if (c == 'v') ; - else if (c >= '0' && c <= '7') { - // process octal escape - if (in.length() > index + 2) { - char buf[4]; - buf[0] = c.toLatin1(); - buf[1] = in[++index].toLatin1(); - buf[2] = in[++index].toLatin1(); - buf[3] = 0; - long int v = strtol(buf, nullptr, 8); - if (v < 0x80) { - output += static_cast(static_cast(v)); - } - else { - state = ERROR; - break; - } - } - else { - state = ERROR; - break; - } - } - else if (c == 'x') { - // hex byte - if (in.length() > index + 2) { - char buf[3]; - buf[0] = in[++index].toLatin1(); - buf[1] = in[++index].toLatin1(); - buf[2] = 0; - long int v = strtol(buf, nullptr, 16); - output += static_cast(static_cast(v)); - } - else { - state = ERROR; - break; - } - } - else if (c == 'u') { - // 4 digit hax unicode codepoint - // hex byte - if (in.length() > index + 4) { - char buf[5]; - buf[0] = in[++index].toLatin1(); - buf[1] = in[++index].toLatin1(); - buf[2] = in[++index].toLatin1(); - buf[3] = in[++index].toLatin1(); - buf[4] = 0; - long int v = strtol(buf, nullptr, 16); - output += static_cast(static_cast(v)); - } - else { - state = ERROR; - break; - } - - } - else if (c == 'U') { - // 8 digit hax unicode codepoint - if (in.length() > index + 8) { - char buf[9]; - buf[0] = in[++index].toLatin1(); - buf[1] = in[++index].toLatin1(); - buf[2] = in[++index].toLatin1(); - buf[3] = in[++index].toLatin1(); - buf[4] = in[++index].toLatin1(); - buf[5] = in[++index].toLatin1(); - buf[6] = in[++index].toLatin1(); - buf[7] = in[++index].toLatin1(); - buf[8] = 0; - uint v = static_cast(strtol(buf, nullptr, 16)); - if (QChar::requiresSurrogates(v)) { - output += QChar(QChar::highSurrogate(v)); - output += QChar(QChar::lowSurrogate(v)); - } - } - else { - state = ERROR; - break; - } - - } - else { - output += c; - } - escape = false; - } - else { - if (c == quote) { - state = WHITESPACE; - ++index; - break; - } - // Is there any language where string literals do not have to be terminated before the end of the line - // Not considering string literals that explicitly allow for multiline strings as these are often raw strings - // and can be copy pasted normally. - else if (c == '\n') { - state = WHITESPACE; - ++index; - break; - } - else if (c == '\\') { - escape = true; - } - else { - output += c; - } - } - ++index; - } - } + while (true) { + // skip all whitespace untill we encounter something else + // we also skip concatenation operators. Note this code is not trying to validate + // for correct syntax so it will quite happily accept many incorrect constructs + // that doesn't matter however as we are just trying to strip everything which is not SQL. + while (index < in.length() && (in[index].isSpace() || in[index] == '+' || in[index] == '.')) ++index; + if (index == in.length()) break; + if (!readOpeningQuote(in, index, quote)) break; + if (!readStringBody(in, index, quote, output)) break; } return output; diff --git a/releasenotes/notes/fix-convert-lang-literals-5c1e8a2f9d4b7360.yaml b/releasenotes/notes/fix-convert-lang-literals-5c1e8a2f9d4b7360.yaml new file mode 100644 index 0000000..f2166bc --- /dev/null +++ b/releasenotes/notes/fix-convert-lang-literals-5c1e8a2f9d4b7360.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Pasting C++ wide and Unicode string literals (``L"..."``, ``U"..."``, ``u"..."`` and ``u8"..."``) into the query tool now converts them to their SQL contents instead of being rejected. + - | + ``\U`` escapes for code points up to U+FFFF are no longer silently dropped when pasting string literals. Invalid ``\U`` code points (above U+10FFFF or in the surrogate range) now stop the conversion, keeping the SQL converted so far. diff --git a/tests/pglabtests/tst_ConvertLangToSqlString.cpp b/tests/pglabtests/tst_ConvertLangToSqlString.cpp index a098c0c..0be100d 100644 --- a/tests/pglabtests/tst_ConvertLangToSqlString.cpp +++ b/tests/pglabtests/tst_ConvertLangToSqlString.cpp @@ -61,6 +61,96 @@ TEST(ConvertLangToSqlString, testSemiColon) ASSERT_EQ(output, expected); } +TEST(ConvertLangToSqlString, testWideLiteralL) +{ + QString in(R"__( L"SELECT" )__"); + QString expected(R"__(SELECT)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testWideLiteralU) +{ + QString in(R"__( U"SELECT" )__"); + QString expected(R"__(SELECT)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUtf16Literal) +{ + QString in(R"__( u"SELECT" )__"); + QString expected(R"__(SELECT)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUtf8Literal) +{ + QString in(R"__( u8"SELECT" )__"); + QString expected(R"__(SELECT)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testPrefixedLiteralConcat) +{ + QString in(R"__( L"A" + "B" )__"); + QString expected(R"__(AB)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUnicodeEscapeBmp) +{ + QString in(R"__( "\U00000041" )__"); + QString expected(R"__(A)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUnicodeEscapeSurrogatePair) +{ + QString in(R"__( "\U0001F600" )__"); + QString expected = QString::fromUcs4(U"\U0001F600"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUnicodeEscapeOutOfRange) +{ + QString in(R"__( "AB\U00110000CD" )__"); + QString expected(R"__(AB)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testUnicodeEscapeSurrogateCodePoint) +{ + QString in(R"__( "AB\U0000D800CD" )__"); + QString expected(R"__(AB)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + +TEST(ConvertLangToSqlString, testHexEscapeInvalidDigit) +{ + QString in(R"__( "AB\x4GCD" )__"); + QString expected(R"__(AB)__"); + + auto output = ConvertLangToSqlString(in); + ASSERT_EQ(output, expected); +} + //TEST(ConvertLangToSqlString, testComment) //{ // QString in(R"__( "SELECT * " // comment