Lexer improvements:

- Convert unquoted idents to lowercase.
- Recognize quoted idents.
- Allow all unicode whitespace characters
- Added UnexpectedSymbol token for unexpected input (otherwise it is just ignored)
- Handle mixed case keywords in the lexer file instead of filtering the stream

pglablib/CaseChangingCharStream.h

@@ -1,83 +0,0 @@
-#pragma once
-
-#include "antlr4-runtime.h"
-#include <QChar>
-
-/// Helper stream for antlr, the lexer does not need to base case sensitive
-/// this is achieved by changing the case of the chars in LA how ever
-/// when the text of a recognized token is captured the getText function
-/// is used which does no case conversion so the parse will receive the original
-/// case.
-class CaseChangingCharStream: public antlr4::CharStream
-{
-public:
-    CaseChangingCharStream(antlr4::CharStream *stream, bool upper)
-        : stream(stream)
-        , upper(upper)
-    {}
-
-    virtual ~CaseChangingCharStream()
-    {}
-
-    virtual void consume() override
-    {
-        stream->consume();
-    }
-
-    virtual size_t LA(ssize_t i) override
-    {
-        int c = stream->LA(i);
-        if (c <= 0)
-            return c;
-
-        if (upper)
-            return QChar::toUpper(c);
-
-        return QChar::toLower(c);
-    }
-
-    virtual std::string getText(const antlr4::misc::Interval &interval) override
-    {
-        return stream->getText(interval);
-    }
-
-    virtual std::string toString() const override
-    {
-        return stream->toString();
-    }
-
-    virtual ssize_t mark() override
-    {
-        return stream->mark();
-    }
-
-    virtual void release(ssize_t marker) override
-    {
-        stream->release(marker);
-    }
-
-    virtual size_t index() override
-    {
-        return stream->index();
-    }
-
-    virtual void seek(size_t index) override
-    {
-        stream->seek(index);
-    }
-
-    virtual size_t size() override
-    {
-        return stream->size();
-    }
-
-    virtual std::string getSourceName() const  override
-    {
-        return stream->getSourceName();
-    }
-
-private:
-    antlr4::CharStream *stream;
-    bool upper;
-
-};

pglablib/PgsqlLexer.g4

@@ -1,7 +1,7 @@
 lexer grammar PgsqlLexer;
 
 @lexer::preinclude {
-#include <memory>
+#include <QString>
 }
 
 
@@ -11,21 +11,61 @@ Dot: '.';
 OpenParen: '(';
 CloseParen: ')';
 
-As: 'AS';
+fragment A : 'a' | 'A';
-By: 'BY';
+fragment B : 'B' | 'b';
-From: 'FROM';
+fragment C : 'C' | 'c';
-Full: 'FULL';
+fragment D : 'D' | 'd';
-Group: 'GROUP';
+fragment E : 'E' | 'e';
-Having: 'HAVING';
+fragment F : 'F' | 'f';
-Join: 'JOIN';
+fragment G : 'G' | 'g';
-Left : 'LEFT';
+fragment H : 'H' | 'h';
-Order : 'ORDER';
+fragment I : 'I' | 'i';
-Right : 'RIGHT';
+fragment J : 'J' | 'j';
-Select: 'SELECT';
+fragment K : 'K' | 'k';
-Where: 'WHERE';
+fragment L : 'L' | 'l';
+fragment M : 'M' | 'm';
+fragment N : 'N' | 'n';
+fragment O : 'O' | 'o';
+fragment P : 'P' | 'p';
+fragment Q : 'Q' | 'q';
+fragment R : 'R' | 'r';
+fragment S : 'S' | 's';
+fragment T : 'T' | 't';
+fragment U : 'U' | 'u';
+fragment V : 'V' | 'v';
+fragment W : 'W' | 'w';
+fragment X : 'X' | 'x';
+fragment Y : 'Y' | 'y';
+fragment Z : 'Z' | 'z';
 
-Ident: [A-Za-z_][A-Za-z_0-9]* ;             // match lower-case identifiers
+As: A S;
+By: B Y;
+From: F R O M;
+Full: F U L L;
+Group: G R O U P;
+Having: H A V I N G;
+Join: J O I N;
+Left : L E F T;
+Order : O R D E R;
+Right : R I G H T;
+Select: S E L E C T;
+Where: W H E R E;
+
+Ident: [\p{Alpha}]~[\p{White_Space}]*
+        {
+            setText(QString::fromStdString(getText()).toLower().toStdString());
+        }
+    | '"' ~["]+ '"' 
+        { 
+            {
+                std::string s = getText();
+                s = s.substr(1, s.length() - 2);
+                setText(s);
+            }
+        };
 IntegerLiteral: [1-9][0-9]*;
-StringLiteral: '\'' ('\'\'' | ~ ('\''))* '\'' { setText(getText().substr(1, getText().length()-2)); };
+StringLiteral: '\'' ('\'\'' | ~['])+ '\'' { setText(getText().substr(1, getText().length()-2)); };
 
-Whitespace : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines
+Whitespace: [\p{White_Space}] -> skip ; // skip spaces, tabs, newlines
+
+UnexpectedSymbol: .;

pglablib/pglablib.pro

@@ -104,7 +104,6 @@ SOURCES += \
     catalog/PgSequenceContainer.cpp
 
 HEADERS += \
-    CaseChangingCharStream.h \
         Pglablib.h \
 	ASyncDBConnection.h \
 	ConnectionConfig.h \

pglablib/sqlparser/Parser.cpp

@@ -8,8 +8,7 @@ Parser::Parser(const std::string &input_string)
 
 Parser::Parser(std::unique_ptr<antlr4::CharStream> stream)
     : InputStream(std::move(stream))
-    , CaseFilter(InputStream.get(), true)
+    , Lexer(InputStream.get())
-    , Lexer(&CaseFilter)
     , TokenStream(&Lexer)
     , AParser(&TokenStream)
 {

pglablib/sqlparser/Parser.h

@@ -2,7 +2,6 @@
 
 #include ".generated/PgsqlLexer.h"
 #include ".generated/PgsqlParser.h"
-#include "CaseChangingCharStream.h"
 #include "ErrorListener.h"
 
 class Parser
@@ -19,7 +18,6 @@ public:
     }
 private:
     std::unique_ptr<antlr4::CharStream> InputStream;
-    CaseChangingCharStream CaseFilter;
     PgsqlLexer Lexer;
     antlr4::CommonTokenStream TokenStream;
     PgsqlParser AParser;

tests/pglabtests/tst_newParser.cpp

@@ -6,6 +6,37 @@
 using namespace testing;
 using namespace sqlast;
 
+TEST(NewSqlLexer, Select)
+{
+    std::string source = "SELECT";
+    antlr4::ANTLRInputStream input(source);
+    PgsqlLexer lexer(&input);
+
+    auto token = lexer.nextToken();
+    ASSERT_EQ(PgsqlLexer::Select, token->getType());
+}
+
+TEST(NewSqlLexer, Ident)
+{
+    std::string source = "Abc";
+    antlr4::ANTLRInputStream input(source);
+    PgsqlLexer lexer(&input);
+
+    auto token = lexer.nextToken();
+    ASSERT_EQ(PgsqlLexer::Ident, token->getType());
+    ASSERT_EQ("abc", token->getText());
+}
+
+TEST(NewSqlLexer, QuotedIdent)
+{
+    std::string source = "\"Abc\"";
+    antlr4::ANTLRInputStream input(source);
+    PgsqlLexer lexer(&input);
+
+    auto token = lexer.nextToken();
+    ASSERT_EQ(PgsqlLexer::Ident, token->getType());
+    ASSERT_EQ("Abc", token->getText());
+}
 
 
 TEST(NewSqlParser, statementList)
@@ -47,3 +78,47 @@ TEST(NewSqlParser, selectList)
     StringLiteral& string_literal = dynamic_cast<StringLiteral&>(si.GetExpression());
     ASSERT_EQ("Tekst", string_literal.GetValue());
 }
+
+TEST(NewSqlParser, selectAliasWithoutAs)
+{
+    std::string input_string = "SELECT 1 a";
+    Parser parser(input_string);
+    std::unique_ptr<sqlast::StatementList> program = parser.Parse();
+
+    ASSERT_TRUE(program != nullptr);
+    ASSERT_EQ(1, program->Count());
+    ASSERT_EQ(0, parser.errorCount());
+
+    SelectStatement &s = dynamic_cast<SelectStatement&>(program->Get(0));
+    SelectList* sl = s.GetSelectList();
+    SelectItem& si = sl->Get(0);
+    ASSERT_EQ("a", si.GetAlias());
+}
+
+
+TEST(NewSqlParser, selectAliasWithAs)
+{
+    std::string input_string = "SELECT 1 AS b";
+    Parser parser(input_string);
+    std::unique_ptr<sqlast::StatementList> program = parser.Parse();
+
+    ASSERT_TRUE(program != nullptr);
+    ASSERT_EQ(1, program->Count());
+    ASSERT_EQ(0, parser.errorCount());
+
+    SelectStatement &s = dynamic_cast<SelectStatement&>(program->Get(0));
+    SelectList* sl = s.GetSelectList();
+    SelectItem& si = sl->Get(0);
+    ASSERT_EQ("b", si.GetAlias());
+}
+
+TEST(NewSqlParser, selectFrom)
+{
+    std::string input_string = "SELECT 1 FROM a";
+    Parser parser(input_string);
+    std::unique_ptr<sqlast::StatementList> program = parser.Parse();
+
+    ASSERT_TRUE(program != nullptr);
+    ASSERT_EQ(1, program->Count());
+    ASSERT_EQ(0, parser.errorCount());
+}