Seperated the lexer from the syntax highlighter for automated testing.

SqlLexer.cpp

@@ -0,0 +1,133 @@
+#include "SqlLexer.h"
+
+SqlLexer::SqlLexer(const QString &block, LexerState currentstate)
+	: m_block(block)
+	, m_state(currentstate)
+{}
+
+QChar SqlLexer::nextChar()
+{
+	QChar result = QChar::Null;
+	if (m_pos < m_block.size()) {
+		result = m_block.at(m_pos++);
+	}
+	return result;
+}
+
+QChar SqlLexer::peekChar()
+{
+	QChar result = QChar::Null;
+	if (m_pos < m_block.size()) {
+		result = m_block.at(m_pos);
+	}
+	return result;
+}
+
+/**
+ * @brief NextBasicToken
+ * @param in
+ * @param ofs
+ * @param start
+ * @param length
+ * @return false when input seems invalid, it will return what it did recognize but something wasn't right, parser should try to recover
+ */
+bool SqlLexer::nextBasicToken(int &startpos, int &length, BasicTokenType &tokentype, QString &out)
+{
+	// Basically chops based on white space
+	// it does also recognize comments and quoted strings/identifiers
+	bool result = false;
+	while (true) {
+		startpos = m_pos;
+		QChar c = nextChar();
+		if (c.isSpace()) {
+			// Just skip whitespace
+		}
+		else if (c == '-' && peekChar() == '-') { // two dashes, start of comment
+			// Loop till end of line or end of block
+			c = nextChar();
+			for (;;) {
+				c = peekChar();
+				if (c != QChar::Null && c != '\n')
+					nextChar();
+				else
+					break;
+			}
+			length = m_pos - startpos;
+			tokentype = BasicTokenType::Comment;
+			return true;
+		}
+		else if (c == '\'') {
+			// Single quoted string so it's an SQL text literal
+			while (true) {
+				c = peekChar();
+				if (c == QChar::Null || c == '\n') {
+					// unexpected end, pretend nothings wrong
+					length = m_pos - startpos;
+					tokentype = BasicTokenType::QuotedString;
+					return true;
+				}
+				else {
+					nextChar();
+					if (c == '\'') {
+						// maybe end of string literal
+						if (peekChar() == '\'') {
+							// Nope, just double quote to escape quote
+							nextChar(); // eat it
+						}
+						else {
+							length = m_pos - startpos;
+							tokentype = BasicTokenType::QuotedString;
+							return true;
+						}
+					}
+				}
+			}
+		}
+		else if (c == '"') {
+			// Double quoted identifier
+			while (true) {
+				c = peekChar();
+				if (c == QChar::Null || c == '\n') {
+					// unexpected end, pretend nothings wrong
+					length = m_pos - startpos;
+					tokentype = BasicTokenType::QuotedIdentifier;
+					return true;
+				}
+				else {
+					nextChar();
+					if (c == '"') {
+						// maybe end of string literal
+						if (peekChar() == '"') {
+							// Nope, just double quote to escape quote
+							nextChar(); // eat it
+						}
+						else {
+							length = m_pos - startpos;
+							tokentype = BasicTokenType::QuotedIdentifier;
+							return true;
+						}
+					}
+				}
+			}
+		}
+		else if (c == QChar::Null) {
+			break;
+		}
+		else {
+			// Undetermined symbol
+			for (;;) {
+				c = peekChar();
+				if (c.isLetterOrNumber() || c == '_')
+					nextChar();
+				else
+					break;
+			}
+			length = m_pos - startpos;
+			tokentype = BasicTokenType::Symbol;
+			QStringRef sr(&m_block, startpos, length);
+			out = sr.toString();
+			return true;
+		}
+	}
+	return false;
+}