cleanup and refactor C# lexer classes

src/Console/AutoCompleter.cs

@@ -133,7 +133,7 @@ namespace UnityExplorer.Console
             {
                 var editor = ConsolePage.Instance.m_codeEditor;
 
-                var textGen = editor.inputText.cachedTextGenerator;
+                var textGen = editor.InputText.cachedTextGenerator;
 
                 //if (textGen.characters.Count < 1)
                 //    return;

src/Console/Lexer/CommentMatch.cs

@@ -3,7 +3,7 @@ using UnityEngine;
 
 namespace UnityExplorer.Console.Lexer
 {
-    public sealed class CommentMatch : Matcher
+    public class CommentMatch : Matcher
     {
         public string lineCommentStart = @"//";
         public string blockCommentStart = @"/*";
@@ -12,9 +12,9 @@ namespace UnityExplorer.Console.Lexer
         public override Color HighlightColor => new Color(0.34f, 0.65f, 0.29f, 1.0f);
         public override IEnumerable<char> StartChars => new char[] { lineCommentStart[0], blockCommentStart[0] };
         public override IEnumerable<char> EndChars => new char[] { blockCommentEnd[0] };
-        public override bool IsImplicitMatch(InputLexer lexer) => IsMatch(lexer, lineCommentStart) || IsMatch(lexer, blockCommentStart);
+        public override bool IsImplicitMatch(CSharpLexer lexer) => IsMatch(lexer, lineCommentStart) || IsMatch(lexer, blockCommentStart);
 
-        private bool IsMatch(InputLexer lexer, string commentType)
+        private bool IsMatch(CSharpLexer lexer, string commentType)
         {
             if (!string.IsNullOrEmpty(commentType))
             {
@@ -32,11 +32,8 @@ namespace UnityExplorer.Console.Lexer
 
                 if (match)
                 {
-                    // Read until end
-                    while (!IsEndLineOrEndFile(lexer, lexer.ReadNext()))
-                    {
-                        ;
-                    }
+                    // Read until end of line or file
+                    while (!IsEndLineOrEndFile(lexer, lexer.ReadNext())) { }
 
                     return true;
                 }
@@ -44,6 +41,6 @@ namespace UnityExplorer.Console.Lexer
             return false;
         }
 
-        private bool IsEndLineOrEndFile(InputLexer lexer, char character) => lexer.EndOfStream || character == '\n' || character == '\r';
+        private bool IsEndLineOrEndFile(CSharpLexer lexer, char character) => lexer.EndOfStream || character == '\n' || character == '\r';
     }
 }

src/Console/Lexer/KeywordMatch.cs

@@ -3,23 +3,22 @@ using UnityEngine;
 
 namespace UnityExplorer.Console.Lexer
 {
-    public sealed class KeywordMatch : Matcher
+    // I use two different KeywordMatch instances (valid and invalid).
+    // This class just contains common implementations.
+    public class KeywordMatch : Matcher
     {
-        public string keywords;
+        public string[] Keywords;
 
         public override Color HighlightColor => highlightColor;
         public Color highlightColor;
 
         private readonly HashSet<string> shortlist = new HashSet<string>();
         private readonly Stack<string> removeList = new Stack<string>();
-        public string[] keywordCache = null;
 
-        public override bool IsImplicitMatch(InputLexer lexer)
+        public override bool IsImplicitMatch(CSharpLexer lexer)
         {
-            BuildKeywordCache();
-
             if (!char.IsWhiteSpace(lexer.Previous) &&
-                !lexer.IsSpecialSymbol(lexer.Previous, SpecialCharacterPosition.End))
+                !lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
             {
                 return false;
             }
@@ -29,11 +28,11 @@ namespace UnityExplorer.Console.Lexer
             int currentIndex = 0;
             char currentChar = lexer.ReadNext();
 
-            for (int i = 0; i < keywordCache.Length; i++)
+            for (int i = 0; i < Keywords.Length; i++)
             {
-                if (keywordCache[i][0] == currentChar)
+                if (Keywords[i][0] == currentChar)
                 {
-                    shortlist.Add(keywordCache[i]);
+                    shortlist.Add(Keywords[i]);
                 }
             }
 
@@ -54,7 +53,7 @@ namespace UnityExplorer.Console.Lexer
                 currentIndex++;
 
                 if (char.IsWhiteSpace(currentChar) ||
-                    lexer.IsSpecialSymbol(currentChar, SpecialCharacterPosition.Start))
+                    lexer.IsSpecialSymbol(currentChar, DelimiterType.Start))
                 {
                     RemoveLongStrings(currentIndex);
                     lexer.Rollback(1);
@@ -94,23 +93,5 @@ namespace UnityExplorer.Console.Lexer
                 shortlist.Remove(removeList.Pop());
             }
         }
-
-        private void BuildKeywordCache()
-        {
-            if (keywordCache == null)
-            {
-                string[] kwSplit = keywords.Split(' ');
-
-                List<string> list = new List<string>();
-                foreach (string kw in kwSplit)
-                {
-                    if (!string.IsNullOrEmpty(kw) && kw.Length > 0)
-                    {
-                        list.Add(kw);
-                    }
-                }
-                keywordCache = list.ToArray();
-            }
-        }
     }
 }

src/Console/Lexer/Matcher.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using UnityExplorer.Unstrip;
 using UnityEngine;
+using System.Linq;
 
 namespace UnityExplorer.Console.Lexer
 {
@@ -9,14 +10,14 @@ namespace UnityExplorer.Console.Lexer
         public abstract Color HighlightColor { get; }
 
         public string HexColor => htmlColor ?? (htmlColor = "<color=#" + HighlightColor.ToHex() + ">");
-        private string htmlColor = null;
+        private string htmlColor;
 
-        public virtual IEnumerable<char> StartChars { get { yield break; } }
-        public virtual IEnumerable<char> EndChars { get { yield break; } }
+        public virtual IEnumerable<char> StartChars => Enumerable.Empty<char>();
+        public virtual IEnumerable<char> EndChars => Enumerable.Empty<char>();
 
-        public abstract bool IsImplicitMatch(InputLexer lexer);
+        public abstract bool IsImplicitMatch(CSharpLexer lexer);
 
-        public bool IsMatch(InputLexer lexer)
+        public bool IsMatch(CSharpLexer lexer)
         {
             if (IsImplicitMatch(lexer))
             {

src/Console/Lexer/NumberMatch.cs

@@ -2,14 +2,14 @@
 
 namespace UnityExplorer.Console.Lexer
 {
-    public sealed class NumberMatch : Matcher
+    public class NumberMatch : Matcher
     {
         public override Color HighlightColor => new Color(0.58f, 0.33f, 0.33f, 1.0f);
 
-        public override bool IsImplicitMatch(InputLexer lexer)
+        public override bool IsImplicitMatch(CSharpLexer lexer)
         {
             if (!char.IsWhiteSpace(lexer.Previous) &&
-                !lexer.IsSpecialSymbol(lexer.Previous, SpecialCharacterPosition.End))
+                !lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
             {
                 return false;
             }

src/Console/Lexer/StringMatch.cs

@@ -3,35 +3,24 @@ using UnityEngine;
 
 namespace UnityExplorer.Console.Lexer
 {
-    public sealed class StringMatch : Matcher
+    public class StringMatch : Matcher
     {
         public override Color HighlightColor => new Color(0.79f, 0.52f, 0.32f, 1.0f);
 
-        public override IEnumerable<char> StartChars { get { yield return '"'; } }
-        public override IEnumerable<char> EndChars { get { yield return '"'; } }
+        public override IEnumerable<char> StartChars => new[] { '"' };
+        public override IEnumerable<char> EndChars => new[] { '"' };
 
-        public override bool IsImplicitMatch(InputLexer lexer)
+        public override bool IsImplicitMatch(CSharpLexer lexer)
         {
             if (lexer.ReadNext() == '"')
             {
-                while (!IsClosingQuoteOrEndFile(lexer, lexer.ReadNext()))
-                {
-                    ;
-                }
+                while (!IsClosingQuoteOrEndFile(lexer, lexer.ReadNext())) { }
 
                 return true;
             }
             return false;
         }
 
-        private bool IsClosingQuoteOrEndFile(InputLexer lexer, char character)
-        {
-            if (lexer.EndOfStream == true ||
-                character == '"')
-            {
-                return true;
-            }
-            return false;
-        }
+        private bool IsClosingQuoteOrEndFile(CSharpLexer lexer, char character) => lexer.EndOfStream || character == '"';
     }
 }

src/Console/Lexer/SymbolMatch.cs

@@ -4,54 +4,32 @@ using UnityEngine;
 
 namespace UnityExplorer.Console.Lexer
 {
-    public sealed class SymbolMatch : Matcher
+    public class SymbolMatch : Matcher
     {
         public override Color HighlightColor => new Color(0.58f, 0.47f, 0.37f, 1.0f);
 
-        public string Symbols => @"[ ] ( ) . ? : + - * / % & | ^ ~ = < > ++ -- && || << >> == != <= >= 
- += -= *= /= %= &= |= ^= <<= >>= -> ?? =>";
+        private readonly string[] symbols = new[]
+        {
+            "[", "]", "(", ")", ".", "?", ":", "+", "-", "*", "/", "%", "&", "|", "^", "~", "=", "<", ">", 
+            "++", "--", "&&", "||", "<<", ">>", "==", "!=", "<=", ">=", "+=", "-=", "*=", "/=", "%=", "&=",
+            "|=", "^=", "<<=", ">>=", "->", "??", "=>",
+        };
 
         private static readonly List<string> shortlist = new List<string>();
         private static readonly Stack<string> removeList = new Stack<string>();
-        private string[] symbolCache = null;
 
-        public override IEnumerable<char> StartChars
-        {
-            get
-            {
-                BuildSymbolCache();
-                foreach (string symbol in symbolCache.Where(x => x.Length > 0))
-                {
-                    yield return symbol[0];
-                }
-            }
-        }
+        public override IEnumerable<char> StartChars => symbols.Select(s => s[0]);
+        public override IEnumerable<char> EndChars => symbols.Select(s => s[0]);
 
-        public override IEnumerable<char> EndChars
-        {
-            get
-            {
-                BuildSymbolCache();
-                foreach (string symbol in symbolCache.Where(x => x.Length > 0))
-                {
-                    yield return symbol[0];
-                }
-            }
-        }
-
-        public override bool IsImplicitMatch(InputLexer lexer)
+        public override bool IsImplicitMatch(CSharpLexer lexer)
         {
             if (lexer == null)
-            {
                 return false;
-            }
-
-            BuildSymbolCache();
 
             if (!char.IsWhiteSpace(lexer.Previous) &&
                 !char.IsLetter(lexer.Previous) &&
                 !char.IsDigit(lexer.Previous) &&
-                !lexer.IsSpecialSymbol(lexer.Previous, SpecialCharacterPosition.End))
+                !lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
             {
                 return false;
             }
@@ -61,18 +39,14 @@ namespace UnityExplorer.Console.Lexer
             int currentIndex = 0;
             char currentChar = lexer.ReadNext();
 
-            for (int i = symbolCache.Length - 1; i >= 0; i--)
+            for (int i = symbols.Length - 1; i >= 0; i--)
             {
-                if (symbolCache[i][0] == currentChar)
-                {
-                    shortlist.Add(symbolCache[i]);
-                }
+                if (symbols[i][0] == currentChar)
+                    shortlist.Add(symbols[i]);
             }
 
             if (shortlist.Count == 0)
-            {
                 return false;
-            }
 
             do
             {
@@ -88,7 +62,7 @@ namespace UnityExplorer.Console.Lexer
                 if (char.IsWhiteSpace(currentChar) ||
                     char.IsLetter(currentChar) ||
                     char.IsDigit(currentChar) ||
-                    lexer.IsSpecialSymbol(currentChar, SpecialCharacterPosition.Start))
+                    lexer.IsSpecialSymbol(currentChar, DelimiterType.Start))
                 {
                     RemoveLongStrings(currentIndex);
                     lexer.Rollback(1);
@@ -128,24 +102,5 @@ namespace UnityExplorer.Console.Lexer
                 shortlist.Remove(removeList.Pop());
             }
         }
-
-        private void BuildSymbolCache()
-        {
-            if (symbolCache != null)
-            {
-                return;
-            }
-
-            string[] symSplit = Symbols.Split(' ');
-            List<string> list = new List<string>();
-            foreach (string sym in symSplit)
-            {
-                if (!string.IsNullOrEmpty(sym) && sym.Length > 0)
-                {
-                    list.Add(sym);
-                }
-            }
-            symbolCache = list.ToArray();
-        }
     }
 }

src/Console/Suggestion.cs

@@ -9,25 +9,22 @@ namespace UnityExplorer.Console
 {
     public struct Suggestion
     {
-        public string Full => Prefix + Addition;
+        public enum Contexts
+        {
+            Namespace,
+            Keyword,
+            Other
+        }
+
+        // ~~~~ Instance ~~~~
 
         public readonly string Prefix;
         public readonly string Addition;
         public readonly Contexts Context;
 
-        public Color TextColor
-        {
-            get
-            {
-                switch (Context)
-                {
-                    case Contexts.Namespace: return Color.grey;
-                    case Contexts.Keyword: return systemBlue;
-                    default: return Color.white;
-                }
-            }
-        }
-        private static readonly Color systemBlue = new Color(80f / 255f, 150f / 255f, 215f / 255f);
+        public string Full => Prefix + Addition;
+
+        public Color TextColor => GetTextColor();
 
         public Suggestion(string addition, string prefix, Contexts type)
         {
@@ -36,16 +33,26 @@ namespace UnityExplorer.Console
             Context = type;
         }
 
-        public enum Contexts
+        private Color GetTextColor()
         {
-            Namespace,
-            Keyword,
-            Other
+            switch (Context)
+            {
+                case Contexts.Namespace: return Color.grey;
+                case Contexts.Keyword: return keywordColor;
+                default: return Color.white;
+            }
         }
 
+        // ~~~~ Static ~~~~
+
         public static HashSet<string> Namespaces => m_namspaces ?? GetNamespaces();
         private static HashSet<string> m_namspaces;
 
+        public static HashSet<string> Keywords => m_keywords ?? (m_keywords = new HashSet<string>(CSharpLexer.validKeywordMatcher.Keywords));
+        private static HashSet<string> m_keywords;
+
+        private static readonly Color keywordColor = new Color(80f / 255f, 150f / 255f, 215f / 255f);
+
         private static HashSet<string> GetNamespaces()
         {
             HashSet<string> set = new HashSet<string>(
@@ -58,25 +65,5 @@ namespace UnityExplorer.Console
 
             IEnumerable<Type> GetTypes(Assembly asm) => asm.TryGetTypes();
         }
-
-        public static HashSet<string> Keywords => m_keywords ?? GetKeywords();
-        private static HashSet<string> m_keywords;
-
-        private static HashSet<string> GetKeywords()
-        {
-            if (CSharpLexer.validKeywordMatcher.keywordCache == null)
-            {
-                return new HashSet<string>();
-            }
-
-            HashSet<string> set = new HashSet<string>();
-
-            foreach (string keyword in CSharpLexer.validKeywordMatcher.keywordCache)
-            {
-                set.Add(keyword);
-            }
-
-            return m_keywords = set;
-        }
     }
 }

src/UI/UIFactory.cs

@@ -78,18 +78,15 @@ namespace UnityExplorer.UI
             if (selectable is Button button)
             {
 #if CPP
-                button.onClick.AddListener(new Action(() =>
-                {
-                    button.OnDeselect(null);
-                }));
+                button.onClick.AddListener(new Action(Deselect));
 #else
 				button.onClick.AddListener(Deselect);
-
+#endif
                 void Deselect()
                 {
 					button.OnDeselect(null);
                 }
-#endif
+
             }
 
             selectable.colors = colors;

src/UnityExplorer.csproj

@@ -355,9 +355,8 @@
     <Compile Include="UI\PageModel\ConsolePage.cs" />
     <Compile Include="Console\AutoCompleter.cs" />
     <Compile Include="Console\CodeEditor.cs" />
-    <Compile Include="Console\CSharpLexer.cs" />
     <Compile Include="Console\Lexer\CommentMatch.cs" />
-    <Compile Include="Console\Lexer\InputLexer.cs" />
+    <Compile Include="Console\CSharpLexer.cs" />
     <Compile Include="Console\Lexer\KeywordMatch.cs" />
     <Compile Include="Console\Lexer\StringMatch.cs" />
     <Compile Include="Console\Lexer\Matcher.cs" />