New syntax highlighter not complete.

- Supports comments
- more efficient as it scans the text block instead of repeatedly searching throught the whole block
- type matching based on catalog (but need to add aliases manually)
- added many keywords

todo:
- heap corruption bug
- symbol stops at special char like parenthese or operator or something similar.

SqlSyntaxHighlighter.cpp

@@ -0,0 +1,240 @@
+#include "SqlSyntaxHighlighter.h"
+
+#include "pgtypecontainer.h"
+
+namespace {
+
+
+	enum class BasicTokenType {
+		None,
+		End, // End of input
+		Symbol, // can be many things, keyword, object name, operator, ..
+		Comment,
+		QuotedString,
+		DollarQuotedString,
+		QuotedIdentifier
+	};
+
+	enum class LexerState {
+		Null,
+		InDollarQuotedString
+	};
+
+
+	class Lexer {
+	private:
+		QString m_block;
+		int m_pos = -1;
+		LexerState m_state;
+	public:
+		Lexer(const QString &block, LexerState currentstate)
+			: m_block(block)
+			, m_state(currentstate)
+		{}
+
+		QChar nextChar()
+		{
+			QChar result = QChar::Null;
+			if (m_pos+1 < m_block.size()) {
+				result = m_block.at(++m_pos);
+			}
+			else {
+				++m_pos;
+			}
+			return result;
+		}
+		QChar peekChar()
+		{
+			QChar result = QChar::Null;
+			if (m_pos+1 < m_block.size()) {
+				result = m_block.at(m_pos+1);
+			}
+			return result;
+		}
+		/**
+		 * @brief NextBasicToken
+		 * @param in
+		 * @param ofs
+		 * @param start
+		 * @param length
+		 * @return false when input seems invalid, it will return what it did recognize but something wasn't right, parser should try to recover
+		 */
+		bool nextBasicToken(int &startpos, int &length, BasicTokenType &tokentype, QString &out)
+		{
+			// Basically chops based on white space
+			// it does also recognize comments and quoted strings/identifiers
+			bool result = false;
+			while (true) {
+				QChar c = nextChar();
+				if (c.isSpace()) {
+					// Just skip whitespace
+				}
+				else if (c == '-' && peekChar() == '-') { // two dashes, start of comment
+					startpos = m_pos;
+					// Loop till end of line or end of block
+					c = nextChar();
+					while (c != QChar::Null && c != '\n') {
+						c = nextChar();
+					}
+					length = m_pos - startpos;
+					tokentype = BasicTokenType::Comment;
+					return true;
+				}
+				else if (c == '\'') {
+					startpos = m_pos;
+					// Single quoted string so it's an SQL text literal
+					while (true) {
+						c = nextChar();
+						if (c == QChar::Null || c == '\n') {
+							// unexpected end, pretend nothings wrong
+							length = m_pos - startpos;
+							tokentype = BasicTokenType::QuotedString;
+							return true;
+						}
+						else if (c == '\'') {
+							// maybe end of string literal
+							if (peekChar() == '\'') {
+								// Nope, just double quote to escape quote
+								nextChar(); // eat it
+							}
+							else {
+								length = m_pos - startpos;
+								tokentype = BasicTokenType::QuotedString;
+								return true;
+							}
+						}
+					}
+				}
+				else if (c == '"') {
+					startpos = m_pos;
+					// Double quoted identifier
+					while (true) {
+						c = nextChar();
+						if (c == QChar::Null || c == '\n') {
+							// unexpected end, pretend nothings wrong
+							length = m_pos - startpos;
+							tokentype = BasicTokenType::QuotedIdentifier;
+							return true;
+						}
+						else if (c == '"') {
+							// maybe end of string literal
+							if (peekChar() == '"') {
+								// Nope, just double quote to escape quote
+								nextChar(); // eat it
+							}
+							else {
+								length = m_pos - startpos;
+								tokentype = BasicTokenType::QuotedIdentifier;
+								return true;
+							}
+						}
+					}
+				}
+				else if (c == QChar::Null) {
+					break;
+				}
+				else {
+					startpos = m_pos;
+					// Undetermined symbol
+					while (!c.isSpace() && c != QChar::Null) {
+						c = nextChar();
+					}
+					length = m_pos - startpos;
+					tokentype = BasicTokenType::Symbol;
+					QStringRef sr(&m_block, startpos, length);
+					out = sr.toString();
+					return true;
+				}
+			}
+			return false;
+		}
+
+	};
+
+
+	t_SymbolSet g_Keywords = {
+		"abort", "after", "aggregate", "alter", "all", "analyze", "and", "any", "as", "asc",
+		"before", "begin", "buffer", "by",
+		"cascade", "case", "cast", "check", "close", "cluster", "collate", "collation", "column", "comment", "commit", "constraint", "copy", "cost", "create",
+		"data", "database", "declare", "default", "deferrable", "deferred", "delete", "desc", "discard", "distinct", "do", "domain", "drop",
+		"elif", "end", "event", "exclude", "execute", "exists", "extenstion",
+		"fetch", "first", "foreign", "from", "function", "full",
+		"global", "grant", "group",
+		"having",
+		"if", "ilike", "immediate", "in", "index", "inherits", "initially", "inner", "insert", "into", "is",
+		"join",
+		"key",
+		"language", "last", "left", "like", "limit", "listen", "local", "lock",
+		"match",
+		"natural", "not", "null", "nulls",
+		"offset", "oids", "on", "or", "order", "outer", "over",
+		"partial", "partition", "prepare", "preserve", "primary", "privileges", "public",
+		"references", "refresh", "reindex", "release", "replace", "reset", "restrict", "revoke", "right", "role", "rollback", "row", "rows", "rule",
+		"savepoint", "schema", "select", "sequence", "server", "set", "show", "simple", "statement",
+		"table", "tablespace", "temp", "temporary", "trigger", "truncate",
+		"unique", "unlisten", "unlogged", "update", "user", "using",
+		"vacuum", "values", "view", "volatile",
+		"when", "where", "with", "wrapper"
+	};
+
+
+
+}
+
+
+SqlSyntaxHighlighter::SqlSyntaxHighlighter(QTextDocument *parent)
+	: QSyntaxHighlighter(parent)
+{
+	m_keywordFormat.setForeground(QColor(32, 32, 192));
+	m_keywordFormat.setFontWeight(QFont::Bold);
+
+	m_commentFormat.setForeground(QColor(64, 64, 64));
+	m_quotedStringFormat.setForeground(QColor(192, 32, 192));
+
+	m_typeFormat.setForeground(QColor(32, 192, 32));
+	m_typeFormat.setFontWeight(QFont::Bold);
+}
+
+SqlSyntaxHighlighter::~SqlSyntaxHighlighter()
+{
+}
+
+void SqlSyntaxHighlighter::setTypes(const PgTypeContainer *types)
+{
+	m_typeNames.clear();
+	for (auto &e : *types) {
+		m_typeNames.insert(e.typname);
+	}
+}
+
+void SqlSyntaxHighlighter::highlightBlock(const QString &text)
+{
+	Lexer lexer(text, LexerState::Null);
+	int startpos, length;
+	BasicTokenType tokentype;
+	QString s;
+	while (lexer.nextBasicToken(startpos, length, tokentype, s)) {
+		switch (tokentype) {
+		case BasicTokenType::None:
+		case BasicTokenType::End: // End of input
+		case BasicTokenType::DollarQuotedString:
+			break;
+		case BasicTokenType::Symbol: // can be many things, keyword, object name, operator, ..
+			if (g_Keywords.count(s.toLower()) > 0) {
+				setFormat(startpos, length, m_keywordFormat);
+			}
+			else if (m_typeNames.count(s.toLower()) > 0) {
+				setFormat(startpos, length, m_typeFormat);
+			}
+			break;
+		case BasicTokenType::Comment:
+			setFormat(startpos, length, m_commentFormat);
+			break;
+		case BasicTokenType::QuotedString:
+			setFormat(startpos, length, m_quotedStringFormat);
+			break;
+		case BasicTokenType::QuotedIdentifier:
+			break;
+		}
+	}
+}

SqlSyntaxHighlighter.h

@@ -0,0 +1,34 @@
+#ifndef SQLSYNTAXHIGHLIGHTER_H
+#define SQLSYNTAXHIGHLIGHTER_H
+
+#include <QSyntaxHighlighter>
+#include <QTextFormat>
+#include <unordered_set>
+#include "util.h"
+
+using t_SymbolSet = std::unordered_set<QString>;
+
+class PgTypeContainer;
+
+class SqlSyntaxHighlighter : public QSyntaxHighlighter
+{
+	Q_OBJECT
+
+public:
+	SqlSyntaxHighlighter(QTextDocument *parent = nullptr);
+	~SqlSyntaxHighlighter();
+
+	void setTypes(const PgTypeContainer *types);
+protected:
+	void highlightBlock(const QString &text) override;
+
+private:
+	QTextCharFormat m_keywordFormat;
+	QTextCharFormat m_commentFormat;
+	QTextCharFormat m_quotedStringFormat;
+	QTextCharFormat m_typeFormat;
+
+	t_SymbolSet m_typeNames;
+};
+
+#endif // SQLSYNTAXHIGHLIGHTER_H

opendatabase.cpp

@@ -37,17 +37,17 @@ bool OpenDatabase::Init()
 	return true;
 }
 
-PgsqlDatabaseCatalogue* OpenDatabase::getCatalogue()
+PgsqlDatabaseCatalogue* OpenDatabase::catalogue()
 {
 	return m_catalogue;
 }
 
 
-TypeSelectionItemModel* OpenDatabase::getTypeSelectionModel()
+TypeSelectionItemModel* OpenDatabase::typeSelectionModel()
 {
 	if (m_typeSelectionModel == nullptr) {
 		m_typeSelectionModel = new TypeSelectionItemModel(nullptr);
-		m_typeSelectionModel->setTypeList(m_catalogue->getTypes());
+		m_typeSelectionModel->setTypeList(m_catalogue->types());
 	}
 	return m_typeSelectionModel;
 }

opendatabase.h

@@ -21,8 +21,8 @@ public:
 	OpenDatabase& operator=(const OpenDatabase &) = delete;
 	~OpenDatabase();
 
-	PgsqlDatabaseCatalogue* getCatalogue();
+	PgsqlDatabaseCatalogue* catalogue();
-	TypeSelectionItemModel* getTypeSelectionModel();
+	TypeSelectionItemModel* typeSelectionModel();
 signals:
 
 public slots:

pglab.pro

@@ -49,7 +49,8 @@ SOURCES += main.cpp\
     ParamTypeDelegate.cpp \
     OpenDatabase.cpp \
     ParamListModel.cpp \
-    MainWindow.cpp
+    MainWindow.cpp \
+    SqlSyntaxHighlighter.cpp
 
 HEADERS  += \
     sqlparser.h \
@@ -85,7 +86,8 @@ HEADERS  += \
     ParamTypeDelegate.h \
     OpenDatabase.h \
     ParamListModel.h \
-    MainWindow.h
+    MainWindow.h \
+    SqlSyntaxHighlighter.h
 
 FORMS    += mainwindow.ui \
     databasewindow.ui \

pgsqldatabasecatalogue.h

@@ -23,7 +23,7 @@ public:
 	void loadAll(Pgsql::Connection &conn);
 	void loadTypes(Pgsql::Connection &conn);
 
-	const PgTypeContainer* getTypes() const { return m_types; }
+	const PgTypeContainer* types() const { return m_types; }
 private:
 	PgTypeContainer *m_types = nullptr;
 };

querytab.cpp

@@ -1,7 +1,7 @@
 #include "querytab.h"
 #include "ui_querytab.h"
 
-#include "sqlhighlighter.h"
+#include "SqlSyntaxHighlighter.h"
 
 #include <QStandardPaths>
 #include <QFileDialog>
@@ -16,6 +16,7 @@
 #include "MainWindow.h"
 #include "OpenDatabase.h"
 #include "pgtypecontainer.h"
+#include "pgsqldatabasecatalogue.h"
 #include "util.h"
 
 QueryTab::QueryTab(MainWindow *win, QWidget *parent) :
@@ -40,10 +41,13 @@ QueryTab::QueryTab(MainWindow *win, QWidget *parent) :
 	font.setFixedPitch(true);
 	font.setPointSize(10);
 	ui->queryEdit->setFont(font);
-	highlighter.reset(new SqlHighlighter(ui->queryEdit->document()));
+	highlighter = new SqlSyntaxHighlighter(ui->queryEdit->document());
 
 	OpenDatabase* open_database = m_win->getDatabase();
-	m_typeDelegate.setTypeSelectionModel(open_database->getTypeSelectionModel());
+	m_typeDelegate.setTypeSelectionModel(open_database->typeSelectionModel());
+
+	auto cat = open_database->catalogue();
+	highlighter->setTypes(cat->types());
 
 	ui->paramTableView->setModel(&m_paramList);
 	ui->paramTableView->setItemDelegateForColumn(1, &m_typeDelegate);

querytab.h

@@ -18,7 +18,7 @@ namespace Ui {
 
 class QTabWidget;
 class MainWindow;
-class SqlHighlighter;
+class SqlSyntaxHighlighter;
 class ExplainRoot;
 class QueryResultModel;
 class QueryExplainModel;
@@ -64,7 +64,7 @@ private:
 
 	Ui::QueryTab *ui;
 	MainWindow *m_win;
-	std::unique_ptr<SqlHighlighter> highlighter;
+	SqlSyntaxHighlighter* highlighter;
 	ConnectionConfig m_config;
 	StopWatch m_stopwatch;
 	ParamListModel m_paramList;

sqlhighlighter.cpp

@@ -74,64 +74,6 @@ SqlHighlighter::SqlHighlighter(QTextDocument *parent)
 //	}
 }
 
-namespace {
-
-	// Advances ofs to first whitespace or end of string, returns false at end of string
-	void skipWhiteSpace(const QString &in, int &ofs)
-	{
-		const int l = in.length();
-		while (ofs < l && in.at(ofs).isSpace())	++ofs;
-	}
-
-	enum class BasicTokenType {
-		None,
-		End, // End of input
-		Symbol, // can be many things, keyword, object name, operator, ..
-		Comment,
-		QuotedString,
-		DollarQuotedString,
-		QuotedIdentifier
-	};
-
-	/**
-	 * @brief NextBasicToken
-	 * @param in
-	 * @param ofs
-	 * @param start
-	 * @param length
-	 * @return false when input seems invalid, it will return what it did recognize but something wasn't right, parser should try to recover
-	 */
-	bool NextBasicToken(const QString &in, int ofs, int &start, int &length, BasicTokenType &tokentype)
-	{
-		// Basically chops based on white space
-		// it does also recognize comments and quoted strings/identifiers
-		bool result = false;
-
-		skipWhiteSpace(in, ofs);
-		const int len = in.length();
-		while (ofs < len) {
-			if (ofs+1 < len && in.at(ofs) == L'-' && in.at(ofs+1) == L'-') {
-				// Start of comment, end of line is end of comment
-
-			}
-			else if (in.at(ofs) == L'\'') {
-				// Start of quoted string
-
-			}
-			else if (in.at(ofs) == L'"') {
-				// Start of quoted identifier
-
-			}
-			else if (in.at(ofs) == L'/' && ofs+1 < len && in.at(ofs+1) == L'*') {
-				// Start of C style comment, scan for end
-			}
-
-		}
-		return result;
-	}
-
-}
-
 void SqlHighlighter::highlightBlock(const QString &text)
 {
 	foreach (const HighlightingRule &rule, highlightingRules) {
@@ -146,3 +88,5 @@ void SqlHighlighter::highlightBlock(const QString &text)
 	setCurrentBlockState(0);
 }
 
+
+

sqlhighlighter.h

@@ -13,7 +13,7 @@ public:
 	SqlHighlighter(QTextDocument *parent = 0);
 
 protected:
-	void highlightBlock(const QString &text) Q_DECL_OVERRIDE;
+	void highlightBlock(const QString &text) override;
 
 private:
 	struct HighlightingRule
@@ -37,4 +37,6 @@ private:
 //     QTextCharFormat multiLineCommentFormat;
 //     QTextCharFormat quotationFormat;
 //     QTextCharFormat functionFormat;
+
 };
+

util.h

@@ -9,4 +9,17 @@ void copySelectionToClipboard(const QTableView *view);
 QString ConvertToMultiLineCString(const QString &in);
 void exportTable(const QTableView *view, QTextStream &out);
 
+namespace std {
+
+  template <>
+  struct hash<QString>
+  {
+	std::size_t operator()(const QString& s) const
+	{
+		return qHash(s);
+	}
+  };
+
+}
+
 #endif // UTIL_H