From fbbe832a05843890e6c14eafcc158cb2cf3e6d5b Mon Sep 17 00:00:00 2001 From: eelke Date: Sun, 3 Apr 2022 12:27:35 +0200 Subject: [PATCH 1/5] Start of new ANTLR4 based parser. Very simple tests pass. --- .gitignore | 2 + common.pri | 2 - core/SqlLexer.h | 4 +- core/SqlParser.cpp | 17 +++++ pglablib/CaseChangingCharStream.h | 83 +++++++++++++++++++++ pglablib/ConnectionConfig.h | 6 +- pglablib/PgsqlLexer.g4 | 31 ++++++++ pglablib/PgsqlParser.g4 | 101 ++++++++++++++++++++++++++ pglablib/build-cpp.bat | 1 + pglablib/pglablib.pro | 44 ++++++++++- pglablib/sqlast/ColumnDefinition.cpp | 9 +++ pglablib/sqlast/ColumnDefinition.h | 28 +++++++ pglablib/sqlast/CreateTable.cpp | 8 ++ pglablib/sqlast/CreateTable.h | 21 ++++++ pglablib/sqlast/Expression.cpp | 8 ++ pglablib/sqlast/Expression.h | 13 ++++ pglablib/sqlast/Literal.cpp | 8 ++ pglablib/sqlast/Literal.h | 13 ++++ pglablib/sqlast/Node.cpp | 8 ++ pglablib/sqlast/Node.h | 24 ++++++ pglablib/sqlast/SelectItem.cpp | 13 ++++ pglablib/sqlast/SelectItem.h | 25 +++++++ pglablib/sqlast/SelectList.cpp | 18 +++++ pglablib/sqlast/SelectList.h | 29 ++++++++ pglablib/sqlast/SelectStatement.cpp | 19 +++++ pglablib/sqlast/SelectStatement.h | 22 ++++++ pglablib/sqlast/Statement.cpp | 10 +++ pglablib/sqlast/Statement.h | 13 ++++ pglablib/sqlast/StatementList.cpp | 24 ++++++ pglablib/sqlast/StatementList.h | 27 +++++++ pglablib/sqlast/StringLiteral.cpp | 7 ++ pglablib/sqlast/StringLiteral.h | 19 +++++ pglablib/sqlast/TypeSpecification.cpp | 8 ++ pglablib/sqlast/TypeSpecification.h | 23 ++++++ pglablib/sqlast/Visitor.cpp | 8 ++ pglablib/sqlast/Visitor.h | 13 ++++ pglablib/sqlast/sqlast.h | 12 +++ pglablib/sqlparser/ErrorListener.cpp | 22 ++++++ pglablib/sqlparser/ErrorListener.h | 21 ++++++ pglablib/sqlparser/Parser.cpp | 21 ++++++ pglablib/sqlparser/Parser.h | 27 +++++++ pglablib/util.cpp | 2 +- tests/pglabtests/pglabtests.pro | 7 +- tests/pglabtests/tst_newParser.cpp | 47 ++++++++++++ 44 files changed, 860 insertions(+), 8 deletions(-) create mode 100644 pglablib/CaseChangingCharStream.h create mode 100644 pglablib/PgsqlLexer.g4 create mode 100644 pglablib/PgsqlParser.g4 create mode 100644 pglablib/build-cpp.bat create mode 100644 pglablib/sqlast/ColumnDefinition.cpp create mode 100644 pglablib/sqlast/ColumnDefinition.h create mode 100644 pglablib/sqlast/CreateTable.cpp create mode 100644 pglablib/sqlast/CreateTable.h create mode 100644 pglablib/sqlast/Expression.cpp create mode 100644 pglablib/sqlast/Expression.h create mode 100644 pglablib/sqlast/Literal.cpp create mode 100644 pglablib/sqlast/Literal.h create mode 100644 pglablib/sqlast/Node.cpp create mode 100644 pglablib/sqlast/Node.h create mode 100644 pglablib/sqlast/SelectItem.cpp create mode 100644 pglablib/sqlast/SelectItem.h create mode 100644 pglablib/sqlast/SelectList.cpp create mode 100644 pglablib/sqlast/SelectList.h create mode 100644 pglablib/sqlast/SelectStatement.cpp create mode 100644 pglablib/sqlast/SelectStatement.h create mode 100644 pglablib/sqlast/Statement.cpp create mode 100644 pglablib/sqlast/Statement.h create mode 100644 pglablib/sqlast/StatementList.cpp create mode 100644 pglablib/sqlast/StatementList.h create mode 100644 pglablib/sqlast/StringLiteral.cpp create mode 100644 pglablib/sqlast/StringLiteral.h create mode 100644 pglablib/sqlast/TypeSpecification.cpp create mode 100644 pglablib/sqlast/TypeSpecification.h create mode 100644 pglablib/sqlast/Visitor.cpp create mode 100644 pglablib/sqlast/Visitor.h create mode 100644 pglablib/sqlast/sqlast.h create mode 100644 pglablib/sqlparser/ErrorListener.cpp create mode 100644 pglablib/sqlparser/ErrorListener.h create mode 100644 pglablib/sqlparser/Parser.cpp create mode 100644 pglablib/sqlparser/Parser.h create mode 100644 tests/pglabtests/tst_newParser.cpp diff --git a/.gitignore b/.gitignore index 645129f..8bbf816 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ srcdoc/ pglabAll.pro.user.4.8-pre1 *.user /pglabAll.pro.user* +.antlr/ +**/.generated/* diff --git a/common.pri b/common.pri index e95c368..5edad44 100644 --- a/common.pri +++ b/common.pri @@ -5,8 +5,6 @@ error( "Use local.pri.sample to create your own local.pri" ) LIBS += -lUser32 -lws2_32 -llibpq CONFIG += c++17 -QMAKE_CXXFLAGS += /std:c++17 - # The following define makes your compiler emit warnings if you use # any feature of Qt which as been marked as deprecated (the exact warnings # depend on your compiler). Please consult the documentation of the diff --git a/core/SqlLexer.h b/core/SqlLexer.h index acc5089..7328100 100644 --- a/core/SqlLexer.h +++ b/core/SqlLexer.h @@ -19,7 +19,9 @@ enum class BasicTokenType { Comma, Cast, WhiteSpace, - NewLine + NewLine, + + LastLexerToken }; enum class LexerState { diff --git a/core/SqlParser.cpp b/core/SqlParser.cpp index 5b63c68..150fc58 100644 --- a/core/SqlParser.cpp +++ b/core/SqlParser.cpp @@ -26,7 +26,24 @@ Keyword isKeyword(const QString &symbol) return Keyword::NotAKeyword; } +/* +Put tokens on a stack +Every time something is put on the stack see if it matches a rule + +The stack needs to contain both tokens from the lexical analyzer as tokens for reductions done by the parser. + +Matching rules, as we need to match against the top of the stack we should match the rules end to start. +Meaning if we have on the stack A B C then we need to consider rules ending with a C + + +*/ + +class StackItem { +public: + int Token; + +}; SqlParser::SqlParser(SqlLexer &lexer) : lexer(lexer) diff --git a/pglablib/CaseChangingCharStream.h b/pglablib/CaseChangingCharStream.h new file mode 100644 index 0000000..7a648c3 --- /dev/null +++ b/pglablib/CaseChangingCharStream.h @@ -0,0 +1,83 @@ +#pragma once + +#include "antlr4-runtime.h" +#include + +/// Helper stream for antlr, the lexer does not need to base case sensitive +/// this is achieved by changing the case of the chars in LA how ever +/// when the text of a recognized token is captured the getText function +/// is used which does no case conversion so the parse will receive the original +/// case. +class CaseChangingCharStream: public antlr4::CharStream +{ +public: + CaseChangingCharStream(antlr4::CharStream *stream, bool upper) + : stream(stream) + , upper(upper) + {} + + virtual ~CaseChangingCharStream() + {} + + virtual void consume() override + { + stream->consume(); + } + + virtual size_t LA(ssize_t i) override + { + int c = stream->LA(i); + if (c <= 0) + return c; + + if (upper) + return QChar::toUpper(c); + + return QChar::toLower(c); + } + + virtual std::string getText(const antlr4::misc::Interval &interval) override + { + return stream->getText(interval); + } + + virtual std::string toString() const override + { + return stream->toString(); + } + + virtual ssize_t mark() override + { + return stream->mark(); + } + + virtual void release(ssize_t marker) override + { + stream->release(marker); + } + + virtual size_t index() override + { + return stream->index(); + } + + virtual void seek(size_t index) override + { + stream->seek(index); + } + + virtual size_t size() override + { + return stream->size(); + } + + virtual std::string getSourceName() const override + { + return stream->getSourceName(); + } + +private: + antlr4::CharStream *stream; + bool upper; + +}; diff --git a/pglablib/ConnectionConfig.h b/pglablib/ConnectionConfig.h index 5e5a9c4..cef7eec 100644 --- a/pglablib/ConnectionConfig.h +++ b/pglablib/ConnectionConfig.h @@ -36,6 +36,7 @@ class ConnectionConfig; class ConnectionNode { public: virtual ~ConnectionNode() = default; + }; class ConnectionGroup: public ConnectionNode { @@ -115,7 +116,10 @@ public: bool dirty() const; void clean(); - bool operator==(QUuid id) const { return m_uuid == id; } + bool operator==(const ConnectionConfig &rhs) const + { + return m_uuid == rhs.m_uuid; + } QString makeLongDescription() const; QByteArray encodedPassword() const; diff --git a/pglablib/PgsqlLexer.g4 b/pglablib/PgsqlLexer.g4 new file mode 100644 index 0000000..699dae0 --- /dev/null +++ b/pglablib/PgsqlLexer.g4 @@ -0,0 +1,31 @@ +lexer grammar PgsqlLexer; + +@lexer::preinclude { +#include +} + + +SemiColon: ';'; +Comma: ','; +Dot: '.'; +OpenParen: '('; +CloseParen: ')'; + +As: 'AS'; +By: 'BY'; +From: 'FROM'; +Full: 'FULL'; +Group: 'GROUP'; +Having: 'HAVING'; +Join: 'JOIN'; +Left : 'LEFT'; +Order : 'ORDER'; +Right : 'RIGHT'; +Select: 'SELECT'; +Where: 'WHERE'; + +Ident: [A-Za-z_][A-Za-z_0-9]* ; // match lower-case identifiers +IntegerLiteral: [1-9][0-9]*; +StringLiteral: '\'' ('\'\'' | ~ ('\''))* '\'' { setText(getText().substr(1, getText().length()-2)); }; + +Whitespace : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines \ No newline at end of file diff --git a/pglablib/PgsqlParser.g4 b/pglablib/PgsqlParser.g4 new file mode 100644 index 0000000..7f817c6 --- /dev/null +++ b/pglablib/PgsqlParser.g4 @@ -0,0 +1,101 @@ +// Define a grammar called postgresql +parser grammar PgsqlParser; + +options { + tokenVocab = PgsqlLexer; +} + +@parser::preinclude { +#include "sqlast/sqlast.h" +} + +@parser::includes { +} + +@parser::members { +} + +main returns [std::unique_ptr program] + : statement_list { $program = std::move($statement_list.result); } + ; + +statement_list returns [std::unique_ptr result] + : { $result = std::make_unique(); } + (statement SemiColon { $result->Add(std::move($statement.result)); } | empty_statement)* + (statement SemiColon? { $result->Add(std::move($statement.result)); } | empty_statement ) + ; + +statement returns [std::unique_ptr result] + : select_stmt { $result = std::move($select_stmt.result); } + ; + +empty_statement + : SemiColon + ; + +select_stmt returns [std::unique_ptr result] + : Select select_list + { + $result = std::make_unique(); + $result->SetSelectList(std::move($select_list.result)); + } + (From from_item (Comma from_item)* )? + (Where condition)? + (Group By group_by)? + (Order By order_by)? + (Having having)? + ; + +from_item + : Ident Dot Ident from_alias? + | Ident from_alias? + | OpenParen select_stmt CloseParen from_alias + ; + +from_alias + : As? Ident (OpenParen Ident (Comma Ident)* CloseParen)? + ; + +condition + : expr + ; + +group_by + : + ; + +order_by + : + ; + +having + : + ; + + +select_list returns [std::unique_ptr result] + : select_item + { + $result = std::make_unique(); + $result->Add(std::move($select_item.result)); + } + (Comma select_item)* { $result->Add(std::move($select_item.result)); } + | + ; + +select_item returns [std::unique_ptr result] + : expr { $result = std::make_unique(std::move($expr.result)); } + (As? Ident { $result->SetAlias($Ident.text); })? + ; + +expr returns [std::unique_ptr result] + : expr Dot Ident + | Ident + | value { $result = std::move($value.result); } + ; + +value returns [std::unique_ptr result] + : IntegerLiteral + | StringLiteral { $result = std::make_unique($StringLiteral.text); } + ; + diff --git a/pglablib/build-cpp.bat b/pglablib/build-cpp.bat new file mode 100644 index 0000000..7c3466f --- /dev/null +++ b/pglablib/build-cpp.bat @@ -0,0 +1 @@ +java -Xmx500M -cp "c:\prog\antlr\antlr-4.9.2-complete.jar;%CLASSPATH%" org.antlr.v4.Tool -Dlanguage=Cpp PgsqlLexer.g4 PgsqlParser.g4 -o sqlparser/.generated -no-listener %2 %3 %4 %5 %6 %7 \ No newline at end of file diff --git a/pglablib/pglablib.pro b/pglablib/pglablib.pro index a835755..32608f8 100644 --- a/pglablib/pglablib.pro +++ b/pglablib/pglablib.pro @@ -4,11 +4,11 @@ # #------------------------------------------------- -QT += widgets core +QT += widgets core concurrent TARGET = pglablib TEMPLATE = lib -CONFIG += staticlib +CONFIG += staticlib no_keywords ! include( ../common.pri ) { error( "Couldn't find the common.pri file!" ) @@ -45,6 +45,23 @@ SOURCES += \ catalog/PgConstraintContainer.cpp \ ParamListJson.cpp \ ParamListModel.cpp \ + sqlast/ColumnDefinition.cpp \ + sqlast/CreateTable.cpp \ + sqlast/Expression.cpp \ + sqlast/Literal.cpp \ + sqlast/Node.cpp \ + sqlast/SelectItem.cpp \ + sqlast/SelectList.cpp \ + sqlast/SelectStatement.cpp \ + sqlast/Statement.cpp \ + sqlast/StatementList.cpp \ + sqlast/StringLiteral.cpp \ + sqlast/TypeSpecification.cpp \ + sqlast/Visitor.cpp \ + sqlparser/.generated/PgsqlLexer.cpp \ + sqlparser/.generated/PgsqlParser.cpp \ + sqlparser/ErrorListener.cpp \ + sqlparser/Parser.cpp \ util.cpp \ SqlFormattingUtils.cpp \ catalog/PgKeywordList.cpp \ @@ -87,6 +104,7 @@ SOURCES += \ catalog/PgSequenceContainer.cpp HEADERS += \ + CaseChangingCharStream.h \ Pglablib.h \ ASyncDBConnection.h \ ConnectionConfig.h \ @@ -113,6 +131,24 @@ HEADERS += \ catalog/PgConstraintContainer.h \ ParamListJson.h \ ParamListModel.h \ + sqlast/ColumnDefinition.h \ + sqlast/CreateTable.h \ + sqlast/Expression.h \ + sqlast/Literal.h \ + sqlast/Node.h \ + sqlast/SelectItem.h \ + sqlast/SelectList.h \ + sqlast/SelectStatement.h \ + sqlast/Statement.h \ + sqlast/StatementList.h \ + sqlast/StringLiteral.h \ + sqlast/TypeSpecification.h \ + sqlast/Visitor.h \ + sqlast/sqlast.h \ + sqlparser/.generated/PgsqlLexer.h \ + sqlparser/.generated/PgsqlParser.h \ + sqlparser/ErrorListener.h \ + sqlparser/Parser.h \ util.h \ SqlFormattingUtils.h \ catalog/PgCatalogTypes.h \ @@ -170,6 +206,10 @@ else:unix:!macx: LIBS += -L$$OUT_PWD/../core/ -lcore INCLUDEPATH += $$PWD/../core DEPENDPATH += $$PWD/../core +INCLUDEPATH += C:\Prog\include\antlr +win32:CONFIG(debug, debug|release): LIBS += -lantlr4-runtimed +else:win32:CONFIG(release, debug|release): LIBS += -lantlr4-runtime + win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../core/release/libcore.a else:win32-g++:CONFIG(debug, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../core/debug/libcore.a else:win32:!win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../core/release/core.lib diff --git a/pglablib/sqlast/ColumnDefinition.cpp b/pglablib/sqlast/ColumnDefinition.cpp new file mode 100644 index 0000000..d300101 --- /dev/null +++ b/pglablib/sqlast/ColumnDefinition.cpp @@ -0,0 +1,9 @@ +#include "ColumnDefinition.h" +#include "TypeSpecification.h" + +using namespace sqlast; + +ColumnDefinition::ColumnDefinition() +{ + +} diff --git a/pglablib/sqlast/ColumnDefinition.h b/pglablib/sqlast/ColumnDefinition.h new file mode 100644 index 0000000..6c98311 --- /dev/null +++ b/pglablib/sqlast/ColumnDefinition.h @@ -0,0 +1,28 @@ +#pragma once + +#include "Node.h" +#include +#include + +namespace sqlast { + +class TypeSpecification; + +/// Defines the details of a table column +/// +/// Constraints are not included here, as we handle constraints can apply to multiple columns +/// and we want to put them all in one place. The UI and SQL generator is allowed to display +/// column specific constraints with the column they belong to. +class ColumnDefinition : public Node +{ +public: + ColumnDefinition(); + +private: + QString name; + std::unique_ptr typeName; + bool notNull = true; + +}; + +} diff --git a/pglablib/sqlast/CreateTable.cpp b/pglablib/sqlast/CreateTable.cpp new file mode 100644 index 0000000..665c7ec --- /dev/null +++ b/pglablib/sqlast/CreateTable.cpp @@ -0,0 +1,8 @@ +#include "CreateTable.h" + +using namespace sqlast; + +CreateTable::CreateTable() +{ + +} diff --git a/pglablib/sqlast/CreateTable.h b/pglablib/sqlast/CreateTable.h new file mode 100644 index 0000000..01afa43 --- /dev/null +++ b/pglablib/sqlast/CreateTable.h @@ -0,0 +1,21 @@ +#pragma once + +#include "Statement.h" +#include + +namespace sqlast { + +class ColumnDefinition; +class TableConstraint; + +class CreateTable: public Statement +{ +public: + CreateTable(); + +private: + + +}; + +} diff --git a/pglablib/sqlast/Expression.cpp b/pglablib/sqlast/Expression.cpp new file mode 100644 index 0000000..52dd30e --- /dev/null +++ b/pglablib/sqlast/Expression.cpp @@ -0,0 +1,8 @@ +#include "Expression.h" + +using namespace sqlast; + +Expression::Expression() +{ + +} diff --git a/pglablib/sqlast/Expression.h b/pglablib/sqlast/Expression.h new file mode 100644 index 0000000..e771a89 --- /dev/null +++ b/pglablib/sqlast/Expression.h @@ -0,0 +1,13 @@ +#pragma once + +#include "Node.h" + +namespace sqlast { + + class Expression: public Node + { + public: + Expression(); + }; + +} diff --git a/pglablib/sqlast/Literal.cpp b/pglablib/sqlast/Literal.cpp new file mode 100644 index 0000000..38f975f --- /dev/null +++ b/pglablib/sqlast/Literal.cpp @@ -0,0 +1,8 @@ +#include "Literal.h" + +using namespace sqlast; + +Literal::Literal() +{ + +} diff --git a/pglablib/sqlast/Literal.h b/pglablib/sqlast/Literal.h new file mode 100644 index 0000000..832abb6 --- /dev/null +++ b/pglablib/sqlast/Literal.h @@ -0,0 +1,13 @@ +#pragma once + +#include "Expression.h" + +namespace sqlast { + + class Literal: public Expression + { + public: + Literal(); + }; + +} diff --git a/pglablib/sqlast/Node.cpp b/pglablib/sqlast/Node.cpp new file mode 100644 index 0000000..e194e6e --- /dev/null +++ b/pglablib/sqlast/Node.cpp @@ -0,0 +1,8 @@ +#include "Node.h" + +using namespace sqlast; + +Node::Node() +{ + +} diff --git a/pglablib/sqlast/Node.h b/pglablib/sqlast/Node.h new file mode 100644 index 0000000..ced5599 --- /dev/null +++ b/pglablib/sqlast/Node.h @@ -0,0 +1,24 @@ +#pragma once + +namespace sqlast { + + class Node { + public: + Node(); + virtual ~Node() = default; + + }; + +} +/* + +- Node + - INSERT + - UPDATE + - DELETE + - SELECT + - WITH + - CTE + + +*/ diff --git a/pglablib/sqlast/SelectItem.cpp b/pglablib/sqlast/SelectItem.cpp new file mode 100644 index 0000000..6066eac --- /dev/null +++ b/pglablib/sqlast/SelectItem.cpp @@ -0,0 +1,13 @@ +#include "SelectItem.h" +#include "Expression.h" + +using namespace sqlast; + +SelectItem::SelectItem(std::unique_ptr expr) + : expression(std::move(expr)) +{} + +void SelectItem::SetAlias(const std::string &alias) +{ + this->alias = alias; +} diff --git a/pglablib/sqlast/SelectItem.h b/pglablib/sqlast/SelectItem.h new file mode 100644 index 0000000..998f4c3 --- /dev/null +++ b/pglablib/sqlast/SelectItem.h @@ -0,0 +1,25 @@ +#pragma once + +#include "Node.h" +#include +#include + +namespace sqlast { + +class Expression; + +class SelectItem : public Node +{ +public: + explicit SelectItem(std::unique_ptr expr); + + Expression& GetExpression() { return *expression; } + + void SetAlias(const std::string &alias); + std::string GetAlias() const { return alias; } +private: + std::unique_ptr expression; + std::string alias; +}; + +} diff --git a/pglablib/sqlast/SelectList.cpp b/pglablib/sqlast/SelectList.cpp new file mode 100644 index 0000000..26ae4d8 --- /dev/null +++ b/pglablib/sqlast/SelectList.cpp @@ -0,0 +1,18 @@ +#include "SelectList.h" +#include "SelectItem.h" + +using namespace sqlast; + +SelectList::SelectList() +{ +} + +void SelectList::Add(std::unique_ptr select_item) +{ + list.push_back(std::move(select_item)); +} + +int SelectList::Count() const +{ + return static_cast(list.size()); +} diff --git a/pglablib/sqlast/SelectList.h b/pglablib/sqlast/SelectList.h new file mode 100644 index 0000000..cf29565 --- /dev/null +++ b/pglablib/sqlast/SelectList.h @@ -0,0 +1,29 @@ +#pragma once + +#include "Node.h" +#include +#include + +namespace sqlast { + + class SelectItem; + + class SelectList : public Node + { + public: + SelectList(); + + void Add(std::unique_ptr select_item); + int Count() const; + + SelectItem& Get(int index) + { + return *list.at(index); + } + private: + using List = std::vector>; + + List list; + }; + +} diff --git a/pglablib/sqlast/SelectStatement.cpp b/pglablib/sqlast/SelectStatement.cpp new file mode 100644 index 0000000..814e54e --- /dev/null +++ b/pglablib/sqlast/SelectStatement.cpp @@ -0,0 +1,19 @@ +#include "SelectStatement.h" +#include "SelectList.h" + +using namespace sqlast; + +SelectStatement::SelectStatement() +{ + +} + +SelectList* SelectStatement::GetSelectList() +{ + return selectList.get(); +} + +void SelectStatement::SetSelectList(std::unique_ptr value) +{ + selectList = std::move(value); +} diff --git a/pglablib/sqlast/SelectStatement.h b/pglablib/sqlast/SelectStatement.h new file mode 100644 index 0000000..c6c9b8d --- /dev/null +++ b/pglablib/sqlast/SelectStatement.h @@ -0,0 +1,22 @@ +#pragma once + +#include "Statement.h" +#include + +namespace sqlast { + + class SelectList; + + class SelectStatement: public Statement + { + public: + SelectStatement(); + + SelectList* GetSelectList(); + void SetSelectList(std::unique_ptr value); + + private: + std::unique_ptr selectList; + }; + +} diff --git a/pglablib/sqlast/Statement.cpp b/pglablib/sqlast/Statement.cpp new file mode 100644 index 0000000..774502f --- /dev/null +++ b/pglablib/sqlast/Statement.cpp @@ -0,0 +1,10 @@ +#include "Statement.h" + +using namespace sqlast; + +Statement::Statement() +{ + +} + + diff --git a/pglablib/sqlast/Statement.h b/pglablib/sqlast/Statement.h new file mode 100644 index 0000000..ee445fa --- /dev/null +++ b/pglablib/sqlast/Statement.h @@ -0,0 +1,13 @@ +#pragma once + +#include "Node.h" + +namespace sqlast { + + class Statement: public Node + { + public: + Statement(); + }; + +} diff --git a/pglablib/sqlast/StatementList.cpp b/pglablib/sqlast/StatementList.cpp new file mode 100644 index 0000000..bf4fcd7 --- /dev/null +++ b/pglablib/sqlast/StatementList.cpp @@ -0,0 +1,24 @@ +#include "StatementList.h" +#include "Statement.h" + + +using namespace sqlast; + +StatementList::StatementList() +{} + +void StatementList::Add(std::unique_ptr &&statement) +{ + statements.push_back(std::move(statement)); +} + +Statement &StatementList::Get(int index) +{ + return *statements[index]; +} + +int StatementList::Count() const +{ + return static_cast(statements.size()); +} + diff --git a/pglablib/sqlast/StatementList.h b/pglablib/sqlast/StatementList.h new file mode 100644 index 0000000..16bd4ee --- /dev/null +++ b/pglablib/sqlast/StatementList.h @@ -0,0 +1,27 @@ +#pragma once + +#include "Node.h" + +#include +#include + +namespace sqlast { + + class Statement; + + class StatementList: public Node + { + public: + StatementList(); + + void Add(std::unique_ptr &&statement); + Statement &Get(int index); + int Count() const; + + private: + using Statements = std::vector>; + + Statements statements; + }; + +} diff --git a/pglablib/sqlast/StringLiteral.cpp b/pglablib/sqlast/StringLiteral.cpp new file mode 100644 index 0000000..d3179eb --- /dev/null +++ b/pglablib/sqlast/StringLiteral.cpp @@ -0,0 +1,7 @@ +#include "StringLiteral.h" + +using namespace sqlast; + +StringLiteral::StringLiteral(const std::string s) + : value(QString::fromStdString(s)) +{} diff --git a/pglablib/sqlast/StringLiteral.h b/pglablib/sqlast/StringLiteral.h new file mode 100644 index 0000000..231c458 --- /dev/null +++ b/pglablib/sqlast/StringLiteral.h @@ -0,0 +1,19 @@ +#pragma once + +#include "Literal.h" +#include +#include + +namespace sqlast { + + class StringLiteral : public Literal + { + public: + explicit StringLiteral(const std::string s); + + QString GetValue() const { return value; } + private: + QString value; + }; + +} diff --git a/pglablib/sqlast/TypeSpecification.cpp b/pglablib/sqlast/TypeSpecification.cpp new file mode 100644 index 0000000..45511dc --- /dev/null +++ b/pglablib/sqlast/TypeSpecification.cpp @@ -0,0 +1,8 @@ +#include "TypeSpecification.h" + +using namespace sqlast; + +TypeSpecification::TypeSpecification() +{ + +} diff --git a/pglablib/sqlast/TypeSpecification.h b/pglablib/sqlast/TypeSpecification.h new file mode 100644 index 0000000..b5ec86c --- /dev/null +++ b/pglablib/sqlast/TypeSpecification.h @@ -0,0 +1,23 @@ +#pragma once + +#include "Node.h" +#include + +namespace sqlast { + +/// These object define not only the base type, but also +/// parameters used with the type +/// Think the precision of numeric, max length of char, array +class TypeSpecification : public Node +{ +public: + TypeSpecification(); + +private: + /// We do not use the PgType from the catalog here as the type used might be defined + /// inside the script and not present yet in the catalog. + QString baseType; + // is_array +}; + +} diff --git a/pglablib/sqlast/Visitor.cpp b/pglablib/sqlast/Visitor.cpp new file mode 100644 index 0000000..16090d5 --- /dev/null +++ b/pglablib/sqlast/Visitor.cpp @@ -0,0 +1,8 @@ +#include "Visitor.h" + +using namespace sqlast; + +Visitor::Visitor() +{ + +} diff --git a/pglablib/sqlast/Visitor.h b/pglablib/sqlast/Visitor.h new file mode 100644 index 0000000..4cf5e7c --- /dev/null +++ b/pglablib/sqlast/Visitor.h @@ -0,0 +1,13 @@ +#pragma once + +namespace sqlast { + +class Visitor +{ +public: + Visitor(); + + +}; + +} diff --git a/pglablib/sqlast/sqlast.h b/pglablib/sqlast/sqlast.h new file mode 100644 index 0000000..e2e04dd --- /dev/null +++ b/pglablib/sqlast/sqlast.h @@ -0,0 +1,12 @@ +#pragma once + +#include "Node.h" +#include "SelectStatement.h" +#include "SelectItem.h" +#include "SelectList.h" +#include "Statement.h" +#include "StatementList.h" +#include "StringLiteral.h" +#undef emit + + diff --git a/pglablib/sqlparser/ErrorListener.cpp b/pglablib/sqlparser/ErrorListener.cpp new file mode 100644 index 0000000..809bce8 --- /dev/null +++ b/pglablib/sqlparser/ErrorListener.cpp @@ -0,0 +1,22 @@ +#include "ErrorListener.h" + + +void ErrorListener::syntaxError(antlr4::Recognizer *recognizer, antlr4::Token *offendingSymbol, size_t line, size_t charPositionInLine, const std::string &msg, std::exception_ptr e) +{ + ++errors; +} + +void ErrorListener::reportAmbiguity(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, const antlrcpp::BitSet &ambigAlts, antlr4::atn::ATNConfigSet *configs) +{ + ++errors; +} + +void ErrorListener::reportAttemptingFullContext(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, antlr4::atn::ATNConfigSet *configs) +{ + ++errors; +} + +void ErrorListener::reportContextSensitivity(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, size_t prediction, antlr4::atn::ATNConfigSet *configs) +{ + ++errors; +} diff --git a/pglablib/sqlparser/ErrorListener.h b/pglablib/sqlparser/ErrorListener.h new file mode 100644 index 0000000..6da2f02 --- /dev/null +++ b/pglablib/sqlparser/ErrorListener.h @@ -0,0 +1,21 @@ +#pragma once +#include "antlr4-runtime.h" + +class ErrorListener : public antlr4::ANTLRErrorListener +{ + // ANTLRErrorListener interface +public: + virtual void syntaxError(antlr4::Recognizer *recognizer, antlr4::Token *offendingSymbol, size_t line, size_t charPositionInLine, const std::string &msg, std::exception_ptr e) override; + virtual void reportAmbiguity(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact, const antlrcpp::BitSet &ambigAlts, antlr4::atn::ATNConfigSet *configs) override; + virtual void reportAttemptingFullContext(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, const antlrcpp::BitSet &conflictingAlts, antlr4::atn::ATNConfigSet *configs) override; + virtual void reportContextSensitivity(antlr4::Parser *recognizer, const antlr4::dfa::DFA &dfa, size_t startIndex, size_t stopIndex, size_t prediction, antlr4::atn::ATNConfigSet *configs) override; + +public: + int errorCount() const + { + return errors; + } + +private: + int errors = 0; +}; diff --git a/pglablib/sqlparser/Parser.cpp b/pglablib/sqlparser/Parser.cpp new file mode 100644 index 0000000..c8a192b --- /dev/null +++ b/pglablib/sqlparser/Parser.cpp @@ -0,0 +1,21 @@ +#include "Parser.h" +#include "antlr4-runtime.h" + + +Parser::Parser(const std::string &input_string) + : InputStream(std::make_unique(input_string)) + , CaseFilter(InputStream.get(), true) + , Lexer(&CaseFilter) + , TokenStream(&Lexer) + , AParser(&TokenStream) +{ + AParser.removeErrorListeners(); + AParser.addErrorListener(&Errors); +} + +std::unique_ptr Parser::Parse() +{ + auto context = AParser.main(); + return std::move(context->program); +} + diff --git a/pglablib/sqlparser/Parser.h b/pglablib/sqlparser/Parser.h new file mode 100644 index 0000000..01b905b --- /dev/null +++ b/pglablib/sqlparser/Parser.h @@ -0,0 +1,27 @@ +#pragma once + +#include ".generated/PgsqlLexer.h" +#include ".generated/PgsqlParser.h" +#include "CaseChangingCharStream.h" +#include "ErrorListener.h" + +class Parser +{ +public: + Parser(const std::string &input_string); + + std::unique_ptr Parse(); + + int errorCount() const + { + return Errors.errorCount(); + } +private: + std::unique_ptr InputStream; + CaseChangingCharStream CaseFilter; + PgsqlLexer Lexer; + antlr4::CommonTokenStream TokenStream; + PgsqlParser AParser; + ErrorListener Errors; +}; + diff --git a/pglablib/util.cpp b/pglablib/util.cpp index bdd6bef..32c9e3d 100644 --- a/pglablib/util.cpp +++ b/pglablib/util.cpp @@ -17,7 +17,7 @@ QString msfloatToHumanReadableString(float ms) if (ms < 1.0f) { val = ms * 1000.f; //result = QString::asprintf("%0.3f", ms * 1000.0f); - unit = u8"μs"; + unit = QString::fromUtf8("μs"); } else if (ms >= 1000.0) { val = ms / 1000.0f; diff --git a/tests/pglabtests/pglabtests.pro b/tests/pglabtests/pglabtests.pro index 73c4847..4a6f85c 100644 --- a/tests/pglabtests/pglabtests.pro +++ b/tests/pglabtests/pglabtests.pro @@ -6,7 +6,7 @@ include(gtest_dependency.pri) TEMPLATE = app CONFIG += console CONFIG -= app_bundle -CONFIG += thread +CONFIG += thread no_keywords CONFIG += qt QT += core widgets @@ -20,6 +20,7 @@ SOURCES += main.cpp \ tst_escapeConnectionStringValue.cpp \ tst_expected.cpp \ tst_SqlLexer.cpp \ + tst_newParser.cpp \ tst_scopeguard.cpp \ tst_CsvWriter.cpp \ tst_PasswordManager.cpp \ @@ -39,6 +40,10 @@ DEPENDPATH += $$PWD/../../core win32:CONFIG(debug, debug|release): LIBS += -lbotand else:win32:CONFIG(release, debug|release): LIBS += -lbotan +INCLUDEPATH += C:\Prog\include\antlr +win32:CONFIG(debug, debug|release): LIBS += -lantlr4-runtimed +else:win32:CONFIG(release, debug|release): LIBS += -lantlr4-runtime + win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../../core/release/libcore.a else:win32-g++:CONFIG(debug, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../../core/debug/libcore.a else:win32:!win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../../core/release/core.lib diff --git a/tests/pglabtests/tst_newParser.cpp b/tests/pglabtests/tst_newParser.cpp new file mode 100644 index 0000000..e721d16 --- /dev/null +++ b/tests/pglabtests/tst_newParser.cpp @@ -0,0 +1,47 @@ +#include +#include +#include "PrintTo_Qt.h" +#include "sqlparser/Parser.h" + +using namespace testing; +using namespace sqlast; + + + +TEST(NewSqlParser, statementList) +{ + std::string input_string = "SEleCT 1; Select 2;"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_TRUE(program != nullptr); + ASSERT_EQ(2, program->Count()); +} + +TEST(NewSqlParser, missingSemi) +{ + std::string input_string = "1"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_EQ(1, parser.errorCount()); +} + +TEST(NewSqlParser, selectList) +{ + std::string input_string = "SEleCT 1, 'Tekst'"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_TRUE(program != nullptr); + ASSERT_EQ(1, program->Count()); + + SelectStatement &s = dynamic_cast(program->Get(0)); + SelectList* sl = s.GetSelectList(); + ASSERT_TRUE(sl != nullptr); + ASSERT_EQ(2, sl->Count()); + + SelectItem& si = sl->Get(1); + StringLiteral& string_literal = dynamic_cast(si.GetExpression()); + ASSERT_EQ("Tekst", string_literal.GetValue()); +} From a0ba9b894f3d646b72b9d53f6e2f832be113df42 Mon Sep 17 00:00:00 2001 From: eelke Date: Sun, 3 Apr 2022 12:30:23 +0200 Subject: [PATCH 2/5] Improve construction of out Parser class. --- pglablib/sqlparser/Parser.cpp | 6 +++++- pglablib/sqlparser/Parser.h | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pglablib/sqlparser/Parser.cpp b/pglablib/sqlparser/Parser.cpp index c8a192b..92e6019 100644 --- a/pglablib/sqlparser/Parser.cpp +++ b/pglablib/sqlparser/Parser.cpp @@ -3,7 +3,11 @@ Parser::Parser(const std::string &input_string) - : InputStream(std::make_unique(input_string)) + : Parser(std::make_unique(input_string)) +{} + +Parser::Parser(std::unique_ptr stream) + : InputStream(std::move(stream)) , CaseFilter(InputStream.get(), true) , Lexer(&CaseFilter) , TokenStream(&Lexer) diff --git a/pglablib/sqlparser/Parser.h b/pglablib/sqlparser/Parser.h index 01b905b..7633252 100644 --- a/pglablib/sqlparser/Parser.h +++ b/pglablib/sqlparser/Parser.h @@ -9,6 +9,7 @@ class Parser { public: Parser(const std::string &input_string); + Parser(std::unique_ptr stream); std::unique_ptr Parse(); From 81f27a6a18129b54525dda56bed40fd255835119 Mon Sep 17 00:00:00 2001 From: eelke Date: Sun, 3 Apr 2022 12:50:08 +0200 Subject: [PATCH 3/5] check there are no unexpected parser errors in tests --- tests/pglabtests/tst_newParser.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pglabtests/tst_newParser.cpp b/tests/pglabtests/tst_newParser.cpp index e721d16..f19bd0c 100644 --- a/tests/pglabtests/tst_newParser.cpp +++ b/tests/pglabtests/tst_newParser.cpp @@ -16,6 +16,7 @@ TEST(NewSqlParser, statementList) ASSERT_TRUE(program != nullptr); ASSERT_EQ(2, program->Count()); + ASSERT_EQ(0, parser.errorCount()); } TEST(NewSqlParser, missingSemi) @@ -35,6 +36,7 @@ TEST(NewSqlParser, selectList) ASSERT_TRUE(program != nullptr); ASSERT_EQ(1, program->Count()); + ASSERT_EQ(0, parser.errorCount()); SelectStatement &s = dynamic_cast(program->Get(0)); SelectList* sl = s.GetSelectList(); From 0da32b916c515ceec0c6a0281102c74b86a1805e Mon Sep 17 00:00:00 2001 From: eelke Date: Sun, 3 Apr 2022 20:09:58 +0200 Subject: [PATCH 4/5] Lexer improvements: - Convert unquoted idents to lowercase. - Recognize quoted idents. - Allow all unicode whitespace characters - Added UnexpectedSymbol token for unexpected input (otherwise it is just ignored) - Handle mixed case keywords in the lexer file instead of filtering the stream --- pglablib/CaseChangingCharStream.h | 83 ------------------------------ pglablib/PgsqlLexer.g4 | 72 ++++++++++++++++++++------ pglablib/pglablib.pro | 1 - pglablib/sqlparser/Parser.cpp | 3 +- pglablib/sqlparser/Parser.h | 2 - tests/pglabtests/tst_newParser.cpp | 75 +++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 104 deletions(-) delete mode 100644 pglablib/CaseChangingCharStream.h diff --git a/pglablib/CaseChangingCharStream.h b/pglablib/CaseChangingCharStream.h deleted file mode 100644 index 7a648c3..0000000 --- a/pglablib/CaseChangingCharStream.h +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#include "antlr4-runtime.h" -#include - -/// Helper stream for antlr, the lexer does not need to base case sensitive -/// this is achieved by changing the case of the chars in LA how ever -/// when the text of a recognized token is captured the getText function -/// is used which does no case conversion so the parse will receive the original -/// case. -class CaseChangingCharStream: public antlr4::CharStream -{ -public: - CaseChangingCharStream(antlr4::CharStream *stream, bool upper) - : stream(stream) - , upper(upper) - {} - - virtual ~CaseChangingCharStream() - {} - - virtual void consume() override - { - stream->consume(); - } - - virtual size_t LA(ssize_t i) override - { - int c = stream->LA(i); - if (c <= 0) - return c; - - if (upper) - return QChar::toUpper(c); - - return QChar::toLower(c); - } - - virtual std::string getText(const antlr4::misc::Interval &interval) override - { - return stream->getText(interval); - } - - virtual std::string toString() const override - { - return stream->toString(); - } - - virtual ssize_t mark() override - { - return stream->mark(); - } - - virtual void release(ssize_t marker) override - { - stream->release(marker); - } - - virtual size_t index() override - { - return stream->index(); - } - - virtual void seek(size_t index) override - { - stream->seek(index); - } - - virtual size_t size() override - { - return stream->size(); - } - - virtual std::string getSourceName() const override - { - return stream->getSourceName(); - } - -private: - antlr4::CharStream *stream; - bool upper; - -}; diff --git a/pglablib/PgsqlLexer.g4 b/pglablib/PgsqlLexer.g4 index 699dae0..55e2ce3 100644 --- a/pglablib/PgsqlLexer.g4 +++ b/pglablib/PgsqlLexer.g4 @@ -1,7 +1,7 @@ lexer grammar PgsqlLexer; @lexer::preinclude { -#include +#include } @@ -11,21 +11,61 @@ Dot: '.'; OpenParen: '('; CloseParen: ')'; -As: 'AS'; -By: 'BY'; -From: 'FROM'; -Full: 'FULL'; -Group: 'GROUP'; -Having: 'HAVING'; -Join: 'JOIN'; -Left : 'LEFT'; -Order : 'ORDER'; -Right : 'RIGHT'; -Select: 'SELECT'; -Where: 'WHERE'; +fragment A : 'a' | 'A'; +fragment B : 'B' | 'b'; +fragment C : 'C' | 'c'; +fragment D : 'D' | 'd'; +fragment E : 'E' | 'e'; +fragment F : 'F' | 'f'; +fragment G : 'G' | 'g'; +fragment H : 'H' | 'h'; +fragment I : 'I' | 'i'; +fragment J : 'J' | 'j'; +fragment K : 'K' | 'k'; +fragment L : 'L' | 'l'; +fragment M : 'M' | 'm'; +fragment N : 'N' | 'n'; +fragment O : 'O' | 'o'; +fragment P : 'P' | 'p'; +fragment Q : 'Q' | 'q'; +fragment R : 'R' | 'r'; +fragment S : 'S' | 's'; +fragment T : 'T' | 't'; +fragment U : 'U' | 'u'; +fragment V : 'V' | 'v'; +fragment W : 'W' | 'w'; +fragment X : 'X' | 'x'; +fragment Y : 'Y' | 'y'; +fragment Z : 'Z' | 'z'; -Ident: [A-Za-z_][A-Za-z_0-9]* ; // match lower-case identifiers +As: A S; +By: B Y; +From: F R O M; +Full: F U L L; +Group: G R O U P; +Having: H A V I N G; +Join: J O I N; +Left : L E F T; +Order : O R D E R; +Right : R I G H T; +Select: S E L E C T; +Where: W H E R E; + +Ident: [\p{Alpha}]~[\p{White_Space}]* + { + setText(QString::fromStdString(getText()).toLower().toStdString()); + } + | '"' ~["]+ '"' + { + { + std::string s = getText(); + s = s.substr(1, s.length() - 2); + setText(s); + } + }; IntegerLiteral: [1-9][0-9]*; -StringLiteral: '\'' ('\'\'' | ~ ('\''))* '\'' { setText(getText().substr(1, getText().length()-2)); }; +StringLiteral: '\'' ('\'\'' | ~['])+ '\'' { setText(getText().substr(1, getText().length()-2)); }; -Whitespace : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines \ No newline at end of file +Whitespace: [\p{White_Space}] -> skip ; // skip spaces, tabs, newlines + +UnexpectedSymbol: .; \ No newline at end of file diff --git a/pglablib/pglablib.pro b/pglablib/pglablib.pro index 32608f8..974b722 100644 --- a/pglablib/pglablib.pro +++ b/pglablib/pglablib.pro @@ -104,7 +104,6 @@ SOURCES += \ catalog/PgSequenceContainer.cpp HEADERS += \ - CaseChangingCharStream.h \ Pglablib.h \ ASyncDBConnection.h \ ConnectionConfig.h \ diff --git a/pglablib/sqlparser/Parser.cpp b/pglablib/sqlparser/Parser.cpp index 92e6019..a6da16f 100644 --- a/pglablib/sqlparser/Parser.cpp +++ b/pglablib/sqlparser/Parser.cpp @@ -8,8 +8,7 @@ Parser::Parser(const std::string &input_string) Parser::Parser(std::unique_ptr stream) : InputStream(std::move(stream)) - , CaseFilter(InputStream.get(), true) - , Lexer(&CaseFilter) + , Lexer(InputStream.get()) , TokenStream(&Lexer) , AParser(&TokenStream) { diff --git a/pglablib/sqlparser/Parser.h b/pglablib/sqlparser/Parser.h index 7633252..173d354 100644 --- a/pglablib/sqlparser/Parser.h +++ b/pglablib/sqlparser/Parser.h @@ -2,7 +2,6 @@ #include ".generated/PgsqlLexer.h" #include ".generated/PgsqlParser.h" -#include "CaseChangingCharStream.h" #include "ErrorListener.h" class Parser @@ -19,7 +18,6 @@ public: } private: std::unique_ptr InputStream; - CaseChangingCharStream CaseFilter; PgsqlLexer Lexer; antlr4::CommonTokenStream TokenStream; PgsqlParser AParser; diff --git a/tests/pglabtests/tst_newParser.cpp b/tests/pglabtests/tst_newParser.cpp index f19bd0c..b8fd06e 100644 --- a/tests/pglabtests/tst_newParser.cpp +++ b/tests/pglabtests/tst_newParser.cpp @@ -6,6 +6,37 @@ using namespace testing; using namespace sqlast; +TEST(NewSqlLexer, Select) +{ + std::string source = "SELECT"; + antlr4::ANTLRInputStream input(source); + PgsqlLexer lexer(&input); + + auto token = lexer.nextToken(); + ASSERT_EQ(PgsqlLexer::Select, token->getType()); +} + +TEST(NewSqlLexer, Ident) +{ + std::string source = "Abc"; + antlr4::ANTLRInputStream input(source); + PgsqlLexer lexer(&input); + + auto token = lexer.nextToken(); + ASSERT_EQ(PgsqlLexer::Ident, token->getType()); + ASSERT_EQ("abc", token->getText()); +} + +TEST(NewSqlLexer, QuotedIdent) +{ + std::string source = "\"Abc\""; + antlr4::ANTLRInputStream input(source); + PgsqlLexer lexer(&input); + + auto token = lexer.nextToken(); + ASSERT_EQ(PgsqlLexer::Ident, token->getType()); + ASSERT_EQ("Abc", token->getText()); +} TEST(NewSqlParser, statementList) @@ -47,3 +78,47 @@ TEST(NewSqlParser, selectList) StringLiteral& string_literal = dynamic_cast(si.GetExpression()); ASSERT_EQ("Tekst", string_literal.GetValue()); } + +TEST(NewSqlParser, selectAliasWithoutAs) +{ + std::string input_string = "SELECT 1 a"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_TRUE(program != nullptr); + ASSERT_EQ(1, program->Count()); + ASSERT_EQ(0, parser.errorCount()); + + SelectStatement &s = dynamic_cast(program->Get(0)); + SelectList* sl = s.GetSelectList(); + SelectItem& si = sl->Get(0); + ASSERT_EQ("a", si.GetAlias()); +} + + +TEST(NewSqlParser, selectAliasWithAs) +{ + std::string input_string = "SELECT 1 AS b"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_TRUE(program != nullptr); + ASSERT_EQ(1, program->Count()); + ASSERT_EQ(0, parser.errorCount()); + + SelectStatement &s = dynamic_cast(program->Get(0)); + SelectList* sl = s.GetSelectList(); + SelectItem& si = sl->Get(0); + ASSERT_EQ("b", si.GetAlias()); +} + +TEST(NewSqlParser, selectFrom) +{ + std::string input_string = "SELECT 1 FROM a"; + Parser parser(input_string); + std::unique_ptr program = parser.Parse(); + + ASSERT_TRUE(program != nullptr); + ASSERT_EQ(1, program->Count()); + ASSERT_EQ(0, parser.errorCount()); +} From 698ccab6abda126f41f150150699b16e563eb4f8 Mon Sep 17 00:00:00 2001 From: eelke Date: Thu, 7 Apr 2022 19:35:29 +0200 Subject: [PATCH 5/5] wip --- pglablib/PgsqlLexer.g4 | 62 +++++++++-------- pglablib/PgsqlParser.g4 | 18 +++++ pglablib/pglablib.pro | 4 ++ .../BuildStandardItemTreeModelNodeVisitor.cpp | 69 +++++++++++++++++++ .../BuildStandardItemTreeModelNodeVisitor.h | 28 ++++++++ pglablib/sqlast/Node.cpp | 6 +- pglablib/sqlast/Node.h | 13 ++++ pglablib/sqlast/NodeVisitor.cpp | 8 +++ pglablib/sqlast/NodeVisitor.h | 27 ++++++++ pglablib/sqlast/SelectItem.cpp | 6 ++ pglablib/sqlast/SelectItem.h | 2 + pglablib/sqlast/SelectList.cpp | 15 +++- pglablib/sqlast/SelectList.h | 11 +-- pglablib/sqlast/SelectStatement.cpp | 6 ++ pglablib/sqlast/SelectStatement.h | 1 + pglablib/sqlast/StatementList.cpp | 7 +- pglablib/sqlast/StatementList.h | 1 + pglablib/sqlast/StringLiteral.cpp | 6 ++ pglablib/sqlast/StringLiteral.h | 1 + tests/pglabtests/tst_newParser.cpp | 11 +++ 20 files changed, 265 insertions(+), 37 deletions(-) create mode 100644 pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.cpp create mode 100644 pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.h create mode 100644 pglablib/sqlast/NodeVisitor.cpp create mode 100644 pglablib/sqlast/NodeVisitor.h diff --git a/pglablib/PgsqlLexer.g4 b/pglablib/PgsqlLexer.g4 index 55e2ce3..91109e7 100644 --- a/pglablib/PgsqlLexer.g4 +++ b/pglablib/PgsqlLexer.g4 @@ -11,45 +11,51 @@ Dot: '.'; OpenParen: '('; CloseParen: ')'; -fragment A : 'a' | 'A'; -fragment B : 'B' | 'b'; -fragment C : 'C' | 'c'; -fragment D : 'D' | 'd'; -fragment E : 'E' | 'e'; -fragment F : 'F' | 'f'; -fragment G : 'G' | 'g'; -fragment H : 'H' | 'h'; -fragment I : 'I' | 'i'; -fragment J : 'J' | 'j'; -fragment K : 'K' | 'k'; -fragment L : 'L' | 'l'; -fragment M : 'M' | 'm'; -fragment N : 'N' | 'n'; -fragment O : 'O' | 'o'; -fragment P : 'P' | 'p'; -fragment Q : 'Q' | 'q'; -fragment R : 'R' | 'r'; -fragment S : 'S' | 's'; -fragment T : 'T' | 't'; -fragment U : 'U' | 'u'; -fragment V : 'V' | 'v'; -fragment W : 'W' | 'w'; -fragment X : 'X' | 'x'; -fragment Y : 'Y' | 'y'; -fragment Z : 'Z' | 'z'; +fragment A : [aA]; +fragment B : [bB]; +fragment C : [cC]; +fragment D : [dD]; +fragment E : [eE]; +fragment F : [fF]; +fragment G : [gG]; +fragment H : [hH]; +fragment I : [iI]; +fragment J : [jJ]; +fragment K : [kK]; +fragment L : [lL]; +fragment M : [mM]; +fragment N : [nN]; +fragment O : [oO]; +fragment P : [pP]; +fragment Q : [qQ]; +fragment R : [rR]; +fragment S : [sS]; +fragment T : [tT]; +fragment U : [uU]; +fragment V : [vV]; +fragment W : [wW]; +fragment X : [xX]; +fragment Y : [yY]; +fragment Z : [zZ]; As: A S; By: B Y; +Cross: C R O S S; From: F R O M; Full: F U L L; Group: G R O U P; Having: H A V I N G; +Inner: I N N E R; Join: J O I N; Left : L E F T; +Natural : N A T U R A L; +On : O N; Order : O R D E R; +Outer : O U T E R; Right : R I G H T; -Select: S E L E C T; -Where: W H E R E; +Select : S E L E C T; +Using : U S I N G; +Where : W H E R E; Ident: [\p{Alpha}]~[\p{White_Space}]* { diff --git a/pglablib/PgsqlParser.g4 b/pglablib/PgsqlParser.g4 index 7f817c6..0b2c947 100644 --- a/pglablib/PgsqlParser.g4 +++ b/pglablib/PgsqlParser.g4 @@ -50,6 +50,24 @@ from_item : Ident Dot Ident from_alias? | Ident from_alias? | OpenParen select_stmt CloseParen from_alias + | from_item (Left|Right|Full) Outer? Join from_item (join_on_condition|join_using_condition) + | from_item Natural (Left|Right|Full) Outer? Join from_item + ; + +join_on_condition + : On + ; + +join_using_condition + : Using OpenParen ident_list CloseParen (As join_using_alias) + ; + +join_using_alias + : + ; + +ident_list + : Ident (Comma Ident)* ; from_alias diff --git a/pglablib/pglablib.pro b/pglablib/pglablib.pro index 974b722..5ae18c3 100644 --- a/pglablib/pglablib.pro +++ b/pglablib/pglablib.pro @@ -45,11 +45,13 @@ SOURCES += \ catalog/PgConstraintContainer.cpp \ ParamListJson.cpp \ ParamListModel.cpp \ + sqlast/BuildStandardItemTreeModelNodeVisitor.cpp \ sqlast/ColumnDefinition.cpp \ sqlast/CreateTable.cpp \ sqlast/Expression.cpp \ sqlast/Literal.cpp \ sqlast/Node.cpp \ + sqlast/NodeVisitor.cpp \ sqlast/SelectItem.cpp \ sqlast/SelectList.cpp \ sqlast/SelectStatement.cpp \ @@ -130,11 +132,13 @@ HEADERS += \ catalog/PgConstraintContainer.h \ ParamListJson.h \ ParamListModel.h \ + sqlast/BuildStandardItemTreeModelNodeVisitor.h \ sqlast/ColumnDefinition.h \ sqlast/CreateTable.h \ sqlast/Expression.h \ sqlast/Literal.h \ sqlast/Node.h \ + sqlast/NodeVisitor.h \ sqlast/SelectItem.h \ sqlast/SelectList.h \ sqlast/SelectStatement.h \ diff --git a/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.cpp b/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.cpp new file mode 100644 index 0000000..6839d9d --- /dev/null +++ b/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.cpp @@ -0,0 +1,69 @@ +#include "BuildStandardItemTreeModelNodeVisitor.h" + +using namespace sqlast; + +namespace { + + template + class AutoRevert { + public: + AutoRevert(T& var, const T newValue) + : variable(var) + , previousValue(var) + { + variable = std::move(newValue); + } + + AutoRevert(T& var, const T&& newValue) + : variable(var) + , previousValue(var) + { + variable = std::move(newValue); + } + + ~AutoRevert() + { + variable = std::move(previousValue); + } + + AutoRevert(const AutoRevert&) = delete; + AutoRevert operator=(const AutoRevert&) = delete; + + private: + T& variable; + T previousValue; + }; + + template + AutoRevert MakeAutoRevert(T& var, const T newValue) + { + return AutoRevert(var, newValue); + } +} + +BuildStandardItemTreeModelNodeVisitor::BuildStandardItemTreeModelNodeVisitor() + : model(std::make_unique()) + , currentParent(model->invisibleRootItem()) +{ +} + +void BuildStandardItemTreeModelNodeVisitor::Visit(SelectItem &selectItem) +{ +} + +void BuildStandardItemTreeModelNodeVisitor::Visit(SelectList &selectList) +{ +} + +void BuildStandardItemTreeModelNodeVisitor::Visit(SelectStatement &selectStatement) +{ + auto item = new QStandardItem("SELECT"); + currentParent->appendRow(item); + auto guard = MakeAutoRevert(currentParent, item); + + +} + +void sqlast::BuildStandardItemTreeModelNodeVisitor::Visit(StatementList &statementList) +{ +} diff --git a/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.h b/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.h new file mode 100644 index 0000000..32dbe63 --- /dev/null +++ b/pglablib/sqlast/BuildStandardItemTreeModelNodeVisitor.h @@ -0,0 +1,28 @@ +#pragma once + +#include "NodeVisitor.h" + +#include +#include + +namespace sqlast { + + class BuildStandardItemTreeModelNodeVisitor : public NodeVisitor + { + public: + BuildStandardItemTreeModelNodeVisitor(); + + // NodeVisitor interface + public: + virtual void Visit(SelectItem &selectItem) override; + virtual void Visit(SelectList &selectList) override; + virtual void Visit(SelectStatement &selectStatement) override; + virtual void Visit(StatementList &statementList) override; + virtual void Visit(StringLiteral &stringLiteral) override; + private: + std::unique_ptr model; + QStandardItem *currentParent; + + }; + +} diff --git a/pglablib/sqlast/Node.cpp b/pglablib/sqlast/Node.cpp index e194e6e..d995fe8 100644 --- a/pglablib/sqlast/Node.cpp +++ b/pglablib/sqlast/Node.cpp @@ -4,5 +4,9 @@ using namespace sqlast; Node::Node() { - +} + +QString Node::ToString() const +{ + return QString::fromUtf8(typeid(*this).name()); } diff --git a/pglablib/sqlast/Node.h b/pglablib/sqlast/Node.h index ced5599..e754d53 100644 --- a/pglablib/sqlast/Node.h +++ b/pglablib/sqlast/Node.h @@ -1,12 +1,25 @@ #pragma once +#include +#include +#include + namespace sqlast { + class NodeVisitor; + class Node { public: Node(); virtual ~Node() = default; + virtual void Accept(NodeVisitor &visitor) = 0; + /// Every derived class that has child nodes should override these + /// to facilitate +// virtual int ChildCount() const { return 0; } +// virtual const Node* GetChild(int index) const { throw std::out_of_range("GetChild"); } + + virtual QString ToString() const; }; } diff --git a/pglablib/sqlast/NodeVisitor.cpp b/pglablib/sqlast/NodeVisitor.cpp new file mode 100644 index 0000000..61c2374 --- /dev/null +++ b/pglablib/sqlast/NodeVisitor.cpp @@ -0,0 +1,8 @@ +#include "NodeVisitor.h" +#include "sqlast/SelectList.h" + +void sqlast::NodeVisitor::VisitSelectListItems(SelectList &selectList) +{ + for (int idx = 0; idx < selectList.Count(); ++idx) + Visit(selectList.Get(idx)); +} diff --git a/pglablib/sqlast/NodeVisitor.h b/pglablib/sqlast/NodeVisitor.h new file mode 100644 index 0000000..baa8c7e --- /dev/null +++ b/pglablib/sqlast/NodeVisitor.h @@ -0,0 +1,27 @@ +#pragma once + +namespace sqlast { + + class SelectItem; + class SelectList; + class SelectStatement; + class StatementList; + class StringLiteral; + + class NodeVisitor + { + public: + virtual ~NodeVisitor() = default; + + virtual void Visit(SelectItem &selectItem) = 0; + virtual void Visit(SelectList &selectList) = 0; + virtual void Visit(SelectStatement &selectStatement) = 0; + virtual void Visit(StatementList &statementList) = 0; + virtual void Visit(StringLiteral &stringLiteral) = 0; + + protected: + void VisitSelectListItems(SelectList &selectList); + + }; + +} diff --git a/pglablib/sqlast/SelectItem.cpp b/pglablib/sqlast/SelectItem.cpp index 6066eac..5f798e0 100644 --- a/pglablib/sqlast/SelectItem.cpp +++ b/pglablib/sqlast/SelectItem.cpp @@ -1,5 +1,6 @@ #include "SelectItem.h" #include "Expression.h" +#include "NodeVisitor.h" using namespace sqlast; @@ -11,3 +12,8 @@ void SelectItem::SetAlias(const std::string &alias) { this->alias = alias; } + +void SelectItem::Accept(NodeVisitor &visitor) +{ + visitor.Visit(*this); +} diff --git a/pglablib/sqlast/SelectItem.h b/pglablib/sqlast/SelectItem.h index 998f4c3..eadfabc 100644 --- a/pglablib/sqlast/SelectItem.h +++ b/pglablib/sqlast/SelectItem.h @@ -17,6 +17,8 @@ public: void SetAlias(const std::string &alias); std::string GetAlias() const { return alias; } + + void Accept(NodeVisitor &visitor) override; private: std::unique_ptr expression; std::string alias; diff --git a/pglablib/sqlast/SelectList.cpp b/pglablib/sqlast/SelectList.cpp index 26ae4d8..695bb05 100644 --- a/pglablib/sqlast/SelectList.cpp +++ b/pglablib/sqlast/SelectList.cpp @@ -1,5 +1,6 @@ #include "SelectList.h" #include "SelectItem.h" +#include "sqlast/NodeVisitor.h" using namespace sqlast; @@ -9,10 +10,20 @@ SelectList::SelectList() void SelectList::Add(std::unique_ptr select_item) { - list.push_back(std::move(select_item)); + selectItems.push_back(std::move(select_item)); } int SelectList::Count() const { - return static_cast(list.size()); + return static_cast(selectItems.size()); +} + +SelectItem &SelectList::Get(int index) +{ + return *selectItems.at(index); +} + +void SelectList::Accept(NodeVisitor &visitor) +{ + visitor.Visit(*this); } diff --git a/pglablib/sqlast/SelectList.h b/pglablib/sqlast/SelectList.h index cf29565..defbf0b 100644 --- a/pglablib/sqlast/SelectList.h +++ b/pglablib/sqlast/SelectList.h @@ -6,6 +6,8 @@ namespace sqlast { + + class SelectItem; class SelectList : public Node @@ -16,14 +18,13 @@ namespace sqlast { void Add(std::unique_ptr select_item); int Count() const; - SelectItem& Get(int index) - { - return *list.at(index); - } + SelectItem& Get(int index); + + void Accept(NodeVisitor &visitor) override; private: using List = std::vector>; - List list; + List selectItems; }; } diff --git a/pglablib/sqlast/SelectStatement.cpp b/pglablib/sqlast/SelectStatement.cpp index 814e54e..e693c06 100644 --- a/pglablib/sqlast/SelectStatement.cpp +++ b/pglablib/sqlast/SelectStatement.cpp @@ -1,5 +1,6 @@ #include "SelectStatement.h" #include "SelectList.h" +#include "NodeVisitor.h" using namespace sqlast; @@ -17,3 +18,8 @@ void SelectStatement::SetSelectList(std::unique_ptr value) { selectList = std::move(value); } + +void SelectStatement::Accept(NodeVisitor &visitor) +{ + visitor.Visit(*this); +} diff --git a/pglablib/sqlast/SelectStatement.h b/pglablib/sqlast/SelectStatement.h index c6c9b8d..6f7708d 100644 --- a/pglablib/sqlast/SelectStatement.h +++ b/pglablib/sqlast/SelectStatement.h @@ -15,6 +15,7 @@ namespace sqlast { SelectList* GetSelectList(); void SetSelectList(std::unique_ptr value); + void Accept(NodeVisitor &visitor) override; private: std::unique_ptr selectList; }; diff --git a/pglablib/sqlast/StatementList.cpp b/pglablib/sqlast/StatementList.cpp index bf4fcd7..6bfe21b 100644 --- a/pglablib/sqlast/StatementList.cpp +++ b/pglablib/sqlast/StatementList.cpp @@ -1,6 +1,6 @@ #include "StatementList.h" #include "Statement.h" - +#include "sqlast/NodeVisitor.h" using namespace sqlast; @@ -22,3 +22,8 @@ int StatementList::Count() const return static_cast(statements.size()); } +void StatementList::Accept(NodeVisitor &visitor) +{ + visitor.Visit(*this); +} + diff --git a/pglablib/sqlast/StatementList.h b/pglablib/sqlast/StatementList.h index 16bd4ee..2d994c4 100644 --- a/pglablib/sqlast/StatementList.h +++ b/pglablib/sqlast/StatementList.h @@ -18,6 +18,7 @@ namespace sqlast { Statement &Get(int index); int Count() const; + void Accept(NodeVisitor &visitor) override; private: using Statements = std::vector>; diff --git a/pglablib/sqlast/StringLiteral.cpp b/pglablib/sqlast/StringLiteral.cpp index d3179eb..7be79c6 100644 --- a/pglablib/sqlast/StringLiteral.cpp +++ b/pglablib/sqlast/StringLiteral.cpp @@ -1,7 +1,13 @@ #include "StringLiteral.h" +#include "NodeVisitor.h" using namespace sqlast; StringLiteral::StringLiteral(const std::string s) : value(QString::fromStdString(s)) {} + +void StringLiteral::Accept(NodeVisitor &visitor) +{ + visitor.Visit(*this); +} diff --git a/pglablib/sqlast/StringLiteral.h b/pglablib/sqlast/StringLiteral.h index 231c458..dc267b1 100644 --- a/pglablib/sqlast/StringLiteral.h +++ b/pglablib/sqlast/StringLiteral.h @@ -12,6 +12,7 @@ namespace sqlast { explicit StringLiteral(const std::string s); QString GetValue() const { return value; } + void Accept(NodeVisitor &visitor) override; private: QString value; }; diff --git a/tests/pglabtests/tst_newParser.cpp b/tests/pglabtests/tst_newParser.cpp index b8fd06e..11d7b7b 100644 --- a/tests/pglabtests/tst_newParser.cpp +++ b/tests/pglabtests/tst_newParser.cpp @@ -38,6 +38,17 @@ TEST(NewSqlLexer, QuotedIdent) ASSERT_EQ("Abc", token->getText()); } +TEST(NewSqlLexer, AcceptNewLineInQuotedIdent) +{ + std::string source = "\"Ab\nc\""; + antlr4::ANTLRInputStream input(source); + PgsqlLexer lexer(&input); + + auto token = lexer.nextToken(); + ASSERT_EQ(PgsqlLexer::Ident, token->getType()); + ASSERT_EQ("Ab\nc", token->getText()); +} + TEST(NewSqlParser, statementList) {