Refactor ConvertLangToSqlString and fix couple of bugs. Verify with unit tests.
This commit is contained in:
eelke
parent
5821c23439
commit
6206d02c20
2 changed files with 248 additions and 218 deletions
|
|
@ -221,43 +221,160 @@ QString ConvertToMultiLineRawCppString(const QString &in)
|
|||
|
||||
namespace {
|
||||
|
||||
class Token {
|
||||
public:
|
||||
enum Type {
|
||||
Code,
|
||||
StringLiteral,
|
||||
SingleLineComment,
|
||||
MultiLineComment
|
||||
};
|
||||
// index points at the first char of the token. On success index is placed after the opening quote and quote is set to '"' or '\''.
|
||||
bool readOpeningQuote(const QString &in, int &index, QChar "e)
|
||||
{
|
||||
QChar c = in[index];
|
||||
if (c == '\'' || c == '\"') {
|
||||
quote = c;
|
||||
++index;
|
||||
return true;
|
||||
}
|
||||
if (c == 'L' || c == 'U') {
|
||||
if (in.length() > index + 1 && in[index + 1] == '\"') {
|
||||
quote = '\"';
|
||||
index += 2;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (c == 'u') {
|
||||
if (in.length() > index + 2 && in[index + 1] == '8' && in[index + 2] == '\"') {
|
||||
quote = '\"';
|
||||
index += 3;
|
||||
return true;
|
||||
}
|
||||
if (in.length() > index + 1 && in[index + 1] == '\"') {
|
||||
quote = '\"';
|
||||
index += 2;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const Type type;
|
||||
/// Depends on type
|
||||
/// Code: literal copy of the matched code
|
||||
/// StringLiteral: the contents of the string literal, escapes have been unescaped
|
||||
/// *Comment, the text in the comment
|
||||
const QString data;
|
||||
int hexDigitValue(QChar c)
|
||||
{
|
||||
if (c >= '0' && c <= '9') return c.unicode() - '0';
|
||||
if (c >= 'a' && c <= 'f') return c.unicode() - 'a' + 10;
|
||||
if (c >= 'A' && c <= 'F') return c.unicode() - 'A' + 10;
|
||||
return -1;
|
||||
}
|
||||
|
||||
Token(Type type, QString data)
|
||||
: type(type)
|
||||
, data(data)
|
||||
{}
|
||||
};
|
||||
// Reads exactly `count` hex digits starting at `first`. Fails on out-of-range or non-hex input.
|
||||
bool parseHexDigits(const QString &in, int first, int count, char32_t &value)
|
||||
{
|
||||
if (in.length() < first + count) return false;
|
||||
value = 0;
|
||||
for (int i = first; i < first + count; ++i) {
|
||||
int digit = hexDigitValue(in[i]);
|
||||
if (digit < 0) return false;
|
||||
value = value * 16 + static_cast<char32_t>(digit);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Tokenizer to get SQL out of a piece of programming language code
|
||||
///
|
||||
/// It works by ignoring most input and only get's triggered by string literals
|
||||
/// and comments. It does return tokens for the code in between just so
|
||||
class ProgLangTokenizer {
|
||||
public:
|
||||
ProgLangTokenizer(const QString &in)
|
||||
: input(in)
|
||||
{}
|
||||
// Appends a code point as UTF-16. Rejects > 0x10FFFF and surrogates D800-DFFF.
|
||||
bool appendCodePoint(QString &out, char32_t cp)
|
||||
{
|
||||
if (cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) return false;
|
||||
if (QChar::requiresSurrogates(cp)) {
|
||||
out += QChar(QChar::highSurrogate(cp));
|
||||
out += QChar(QChar::lowSurrogate(cp));
|
||||
}
|
||||
else {
|
||||
out += QChar(static_cast<char16_t>(cp));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
const QString input;
|
||||
int position = 0;
|
||||
// index points at the first octal digit. On success index points at the last consumed digit.
|
||||
bool decodeOctalEscape(const QString &in, int &index, QString &out)
|
||||
{
|
||||
if (in.length() <= index + 2) return false;
|
||||
int value = 0;
|
||||
for (int i = index; i <= index + 2; ++i) {
|
||||
QChar digit = in[i];
|
||||
if (digit < '0' || digit > '7') return false;
|
||||
value = value * 8 + (digit.unicode() - '0');
|
||||
}
|
||||
if (value >= 0x80) return false;
|
||||
out += static_cast<QChar>(static_cast<char>(value));
|
||||
index += 2;
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
// index points at the char after the backslash. On success index points at the last consumed char.
|
||||
bool decodeEscape(const QString &in, int &index, QString &out)
|
||||
{
|
||||
switch (in[index].unicode()) {
|
||||
case 'a':
|
||||
out += '\x07';
|
||||
break;
|
||||
case 'b':
|
||||
out += '\x08';
|
||||
break;
|
||||
case 'f':
|
||||
out += '\x0c';
|
||||
break;
|
||||
case 'n':
|
||||
out += '\n';
|
||||
break;
|
||||
case 'r':
|
||||
case 'v':
|
||||
break;
|
||||
case '0': case '1': case '2': case '3':
|
||||
case '4': case '5': case '6': case '7':
|
||||
return decodeOctalEscape(in, index, out);
|
||||
case 'x': {
|
||||
char32_t value = 0;
|
||||
if (!parseHexDigits(in, index + 1, 2, value)) return false;
|
||||
out += static_cast<QChar>(static_cast<char>(value));
|
||||
index += 2;
|
||||
break;
|
||||
}
|
||||
case 'u': {
|
||||
char32_t value = 0;
|
||||
if (!parseHexDigits(in, index + 1, 4, value)) return false;
|
||||
out += QChar(static_cast<ushort>(value));
|
||||
index += 4;
|
||||
break;
|
||||
}
|
||||
case 'U': {
|
||||
char32_t value = 0;
|
||||
if (!parseHexDigits(in, index + 1, 8, value)) return false;
|
||||
if (!appendCodePoint(out, value)) return false;
|
||||
index += 8;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
out += in[index];
|
||||
break;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// index points at the first char after the opening quote. Stops at the closing quote (consumed), at newline (consumed) or at end of input.
|
||||
bool readStringBody(const QString &in, int &index, QChar quote, QString &out)
|
||||
{
|
||||
while (index < in.length()) {
|
||||
QChar c = in[index];
|
||||
if (c == quote || c == '\n') {
|
||||
++index;
|
||||
return true;
|
||||
}
|
||||
if (c == '\\') {
|
||||
++index;
|
||||
if (index == in.length()) return true;
|
||||
if (!decodeEscape(in, index, out)) return false;
|
||||
}
|
||||
else {
|
||||
out += c;
|
||||
}
|
||||
++index;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -275,195 +392,18 @@ QString ConvertLangToSqlString(const QString &in)
|
|||
// until we reach matching end quote
|
||||
// skip whitespace and + or .
|
||||
|
||||
|
||||
QString output;
|
||||
|
||||
enum {
|
||||
WHITESPACE,
|
||||
PREFIX,
|
||||
IN_STRING,
|
||||
SingleLineComment,
|
||||
END,
|
||||
ERROR
|
||||
} state = WHITESPACE;
|
||||
int index = 0;
|
||||
QChar quote = '\0';
|
||||
while (state != ERROR && state != END && index < in.length()) {
|
||||
if (state == WHITESPACE) {
|
||||
// skip all whitespace untill we encounter something else
|
||||
// we also skip concatenation operators. Note this code is not trying to validate
|
||||
// for correct syntax so it will quite happily accept many incorrect constructs
|
||||
// that doesn't matter however as we are just trying to strip everything which is not SQL.
|
||||
while (index < in.length() && (in[index].isSpace() || in[index] == '+' || in[index] == '.')) ++index;
|
||||
if (index == in.length()) {
|
||||
state = END;
|
||||
break;
|
||||
}
|
||||
// Assume quotes can vary
|
||||
if (in[index] == '\'' || in[index] == '\"') {
|
||||
quote = in[index];
|
||||
++index;
|
||||
state = IN_STRING;
|
||||
}
|
||||
else {
|
||||
state = PREFIX;
|
||||
}
|
||||
}
|
||||
else if (state == PREFIX) {
|
||||
auto c = in[index];
|
||||
if (c == 'L' || c == 'U') {
|
||||
// C++ prefix expect C++ double quote
|
||||
if (in.length() > index+ 1 && in[index+1] == '"') {
|
||||
index += 2;
|
||||
state = IN_STRING;
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (c == 'u') {
|
||||
// C++ prefix expect C++ double quote
|
||||
if (in.length() > index+ 2 && in[index+1] == '8' && in[index+2] == '"') {
|
||||
index += 3;
|
||||
state = IN_STRING;
|
||||
}
|
||||
else if (in.length() > index+ 1 && in[index+1] == '"') {
|
||||
index += 2;
|
||||
state = IN_STRING;
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (state == IN_STRING) {
|
||||
// scan contents of string and process any escapes encountered
|
||||
bool escape = false;
|
||||
while (state != ERROR && index < in.length()) {
|
||||
QChar c = in[index];
|
||||
if (escape) {
|
||||
if (c == 'a') output += '\x07';
|
||||
else if (c == 'b') output += '\x08';
|
||||
else if (c == 'f') output += '\x0c';
|
||||
else if (c == 'n') output += '\n';
|
||||
else if (c == 'r') ;
|
||||
else if (c == 'v') ;
|
||||
else if (c >= '0' && c <= '7') {
|
||||
// process octal escape
|
||||
if (in.length() > index + 2) {
|
||||
char buf[4];
|
||||
buf[0] = c.toLatin1();
|
||||
buf[1] = in[++index].toLatin1();
|
||||
buf[2] = in[++index].toLatin1();
|
||||
buf[3] = 0;
|
||||
long int v = strtol(buf, nullptr, 8);
|
||||
if (v < 0x80) {
|
||||
output += static_cast<QChar>(static_cast<char>(v));
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (c == 'x') {
|
||||
// hex byte
|
||||
if (in.length() > index + 2) {
|
||||
char buf[3];
|
||||
buf[0] = in[++index].toLatin1();
|
||||
buf[1] = in[++index].toLatin1();
|
||||
buf[2] = 0;
|
||||
long int v = strtol(buf, nullptr, 16);
|
||||
output += static_cast<QChar>(static_cast<char>(v));
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (c == 'u') {
|
||||
// 4 digit hax unicode codepoint
|
||||
// hex byte
|
||||
if (in.length() > index + 4) {
|
||||
char buf[5];
|
||||
buf[0] = in[++index].toLatin1();
|
||||
buf[1] = in[++index].toLatin1();
|
||||
buf[2] = in[++index].toLatin1();
|
||||
buf[3] = in[++index].toLatin1();
|
||||
buf[4] = 0;
|
||||
long int v = strtol(buf, nullptr, 16);
|
||||
output += static_cast<QChar>(static_cast<ushort>(v));
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
else if (c == 'U') {
|
||||
// 8 digit hax unicode codepoint
|
||||
if (in.length() > index + 8) {
|
||||
char buf[9];
|
||||
buf[0] = in[++index].toLatin1();
|
||||
buf[1] = in[++index].toLatin1();
|
||||
buf[2] = in[++index].toLatin1();
|
||||
buf[3] = in[++index].toLatin1();
|
||||
buf[4] = in[++index].toLatin1();
|
||||
buf[5] = in[++index].toLatin1();
|
||||
buf[6] = in[++index].toLatin1();
|
||||
buf[7] = in[++index].toLatin1();
|
||||
buf[8] = 0;
|
||||
uint v = static_cast<uint>(strtol(buf, nullptr, 16));
|
||||
if (QChar::requiresSurrogates(v)) {
|
||||
output += QChar(QChar::highSurrogate(v));
|
||||
output += QChar(QChar::lowSurrogate(v));
|
||||
}
|
||||
}
|
||||
else {
|
||||
state = ERROR;
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
output += c;
|
||||
}
|
||||
escape = false;
|
||||
}
|
||||
else {
|
||||
if (c == quote) {
|
||||
state = WHITESPACE;
|
||||
++index;
|
||||
break;
|
||||
}
|
||||
// Is there any language where string literals do not have to be terminated before the end of the line
|
||||
// Not considering string literals that explicitly allow for multiline strings as these are often raw strings
|
||||
// and can be copy pasted normally.
|
||||
else if (c == '\n') {
|
||||
state = WHITESPACE;
|
||||
++index;
|
||||
break;
|
||||
}
|
||||
else if (c == '\\') {
|
||||
escape = true;
|
||||
}
|
||||
else {
|
||||
output += c;
|
||||
}
|
||||
}
|
||||
++index;
|
||||
}
|
||||
}
|
||||
while (true) {
|
||||
// skip all whitespace untill we encounter something else
|
||||
// we also skip concatenation operators. Note this code is not trying to validate
|
||||
// for correct syntax so it will quite happily accept many incorrect constructs
|
||||
// that doesn't matter however as we are just trying to strip everything which is not SQL.
|
||||
while (index < in.length() && (in[index].isSpace() || in[index] == '+' || in[index] == '.')) ++index;
|
||||
if (index == in.length()) break;
|
||||
if (!readOpeningQuote(in, index, quote)) break;
|
||||
if (!readStringBody(in, index, quote, output)) break;
|
||||
}
|
||||
|
||||
return output;
|
||||
|
|
|
|||
|
|
@ -61,6 +61,96 @@ TEST(ConvertLangToSqlString, testSemiColon)
|
|||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testWideLiteralL)
|
||||
{
|
||||
QString in(R"__( L"SELECT" )__");
|
||||
QString expected(R"__(SELECT)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testWideLiteralU)
|
||||
{
|
||||
QString in(R"__( U"SELECT" )__");
|
||||
QString expected(R"__(SELECT)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUtf16Literal)
|
||||
{
|
||||
QString in(R"__( u"SELECT" )__");
|
||||
QString expected(R"__(SELECT)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUtf8Literal)
|
||||
{
|
||||
QString in(R"__( u8"SELECT" )__");
|
||||
QString expected(R"__(SELECT)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testPrefixedLiteralConcat)
|
||||
{
|
||||
QString in(R"__( L"A" + "B" )__");
|
||||
QString expected(R"__(AB)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUnicodeEscapeBmp)
|
||||
{
|
||||
QString in(R"__( "\U00000041" )__");
|
||||
QString expected(R"__(A)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUnicodeEscapeSurrogatePair)
|
||||
{
|
||||
QString in(R"__( "\U0001F600" )__");
|
||||
QString expected = QString::fromUcs4(U"\U0001F600");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUnicodeEscapeOutOfRange)
|
||||
{
|
||||
QString in(R"__( "AB\U00110000CD" )__");
|
||||
QString expected(R"__(AB)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testUnicodeEscapeSurrogateCodePoint)
|
||||
{
|
||||
QString in(R"__( "AB\U0000D800CD" )__");
|
||||
QString expected(R"__(AB)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
TEST(ConvertLangToSqlString, testHexEscapeInvalidDigit)
|
||||
{
|
||||
QString in(R"__( "AB\x4GCD" )__");
|
||||
QString expected(R"__(AB)__");
|
||||
|
||||
auto output = ConvertLangToSqlString(in);
|
||||
ASSERT_EQ(output, expected);
|
||||
}
|
||||
|
||||
//TEST(ConvertLangToSqlString, testComment)
|
||||
//{
|
||||
// QString in(R"__( "SELECT * " // comment
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue