Added page with the types (no details yet)

pglablib/util.cpp

@@ -211,6 +211,48 @@ QString ConvertToMultiLineRawCppString(const QString &in)
     return out;
 }
 
+namespace {
+
+    class Token {
+    public:
+        enum Type {
+            Code,
+            StringLiteral,
+            SingleLineComment,
+            MultiLineComment
+        };
+
+        const Type type;
+        /// Depends on type
+        /// Code: literal copy of the matched code
+        /// StringLiteral: the contents of the string literal, escapes have been unescaped
+        /// *Comment, the text in the comment
+        const QString data;
+
+        Token(Type type, QString data)
+            : type(type)
+            , data(data)
+        {}
+    };
+
+    /// Tokenizer to get SQL out of a piece of programming language code
+    ///
+    /// It works by ignoring most input and only get's triggered by string literals
+    /// and comments. It does return tokens for the code in between just so
+    class ProgLangTokenizer {
+    public:
+        ProgLangTokenizer(const QString &in)
+            : input(in)
+        {}
+
+    private:
+        const QString input;
+        int position = 0;
+
+    };
+
+}
+
 QString ConvertLangToSqlString(const QString &in)
 {
 	// Assume mostly C++ for now but allow some other things like
@@ -232,6 +274,7 @@ QString ConvertLangToSqlString(const QString &in)
 		WHITESPACE,
 		PREFIX,
 		IN_STRING,
+        SingleLineComment,
 		END,
 		ERROR
 	} state = WHITESPACE;