bugfixes, made pckedit more responsive

codewitch-honey-crisis · Aug 21, 2019 · ed86c1a · ed86c1a
1 parent d8474ae
commit ed86c1a
Show file tree

Hide file tree

Showing 23 changed files with 477 additions and 448 deletions.
diff --git a/fa/LexDocument.cs b/fa/LexDocument.cs
@@ -130,7 +130,7 @@ public CharFA<string> ToLexer(IProgress<FAProgress> progress=null)
 			}
 			return result as CharFA<string>;
 		}
-		public IEnumerable<Token> ToTokenizer(IEnumerable<char> input, IEnumerable<string> symbols = null,IProgress<FAProgress> progress=null)
+		public ITokenizer ToTokenizer(IEnumerable<char> input, IEnumerable<string> symbols = null,IProgress<FAProgress> progress=null)
 		{
 			var lexer = ToLexer(progress);
 			var ii = 0;

diff --git a/lalr1/Lalr1.cs b/lalr1/Lalr1.cs
@@ -8,14 +8,14 @@ namespace Pck
 	using Lrfa = FA<string, ICollection<Lalr1.LRItem>>;
 	public static class Lalr1
 	{
-		public static Lalr1Parser ToLalr1Parser(this CfgDocument cfg,IEnumerable<Token> tokenizer,IProgress<CfgLalr1Progress> progress=null)
+		public static Lalr1Parser ToLalr1Parser(this CfgDocument cfg,ITokenizer tokenizer,IProgress<CfgLalr1Progress> progress=null)
 		{
 			Lalr1Parser parser;
 			var res = TryToLalr1Parser(cfg, out parser,tokenizer,progress);
 			CfgException.ThrowIfErrors(res);
 			return parser;
 		}
-		public static IList<CfgMessage> TryToLalr1Parser(this CfgDocument cfg, out Lalr1Parser parser,IEnumerable<Token> tokenizer = null,IProgress<CfgLalr1Progress> progress=null)
+		public static IList<CfgMessage> TryToLalr1Parser(this CfgDocument cfg, out Lalr1Parser parser,ITokenizer tokenizer = null,IProgress<CfgLalr1Progress> progress=null)
 		{
 			CfgLalr1ParseTable parseTable;
 			var result = TryToLalr1ParseTable(cfg, progress, out parseTable);

diff --git a/lalr1/Lalr1DebugParser.cs b/lalr1/Lalr1DebugParser.cs
@@ -10,6 +10,7 @@ public class Lalr1DebugParser : Lalr1Parser
 		CfgLalr1ParseTable _parseTable;
 		Token _token;
 		Token _errorToken;
+		ITokenizer _tokenizer;
 		IEnumerator<Token> _tokenEnum;
 		HashSet<string> _hidden;
 		HashSet<string> _collapsed;
@@ -18,12 +19,14 @@ public class Lalr1DebugParser : Lalr1Parser
 		LRNodeType _nodeType;
 		Stack<int> _stack;
 		string[] _ruleDef;
-		public Lalr1DebugParser(CfgDocument cfg, IEnumerable<Token> tokenizer, CfgLalr1ParseTable parseTable = null)
+		public Lalr1DebugParser(CfgDocument cfg, ITokenizer tokenizer, CfgLalr1ParseTable parseTable = null)
 		{
 			_cfg = cfg;
 			_PopulateAttrs();
 			_stack = new Stack<int>();
-			_tokenEnum = tokenizer.GetEnumerator();
+			_tokenizer = tokenizer;
+			if(null!=tokenizer)
+				_tokenEnum = tokenizer.GetEnumerator();
 			_parseTable = parseTable ?? cfg.ToLalr1ParseTable();
 			_nodeType = LRNodeType.Initial;
 		}
@@ -288,11 +291,22 @@ public override void Restart()
 			_stack.Clear();
 			_nodeType = LRNodeType.Initial;
 		}
-		public override void Restart(IEnumerable<Token> tokenizer)
+		public override void Restart(ITokenizer tokenizer)
 		{
 			Close();
+			_tokenizer = null;
 			if (null != tokenizer)
+			{
+				_tokenizer = tokenizer;
 				_tokenEnum = tokenizer.GetEnumerator();
+			}
+			_nodeType = LRNodeType.Initial;
+		}
+		public override void Restart(IEnumerable<char> input)
+		{
+			Close();
+			_tokenizer.Restart(input);
+			_tokenEnum = _tokenizer.GetEnumerator();
 		}
 		void _Panic()
 		{

diff --git a/ll1/LL1.cs b/ll1/LL1.cs
@@ -6,14 +6,14 @@ namespace Pck
 {
 	public static class LL1
 	{
-		public static LL1Parser ToLL1Parser(this CfgDocument cfg, IEnumerable<Token> tokenizer = null, IProgress<CfgLL1Progress> progress = null)
+		public static LL1Parser ToLL1Parser(this CfgDocument cfg, ITokenizer tokenizer = null, IProgress<CfgLL1Progress> progress = null)
 		{
 			LL1Parser parser;
 			var res = TryToLL1Parser(cfg, out parser, tokenizer, progress);
 			CfgException.ThrowIfErrors(res);
 			return parser;
 		}
-		public static IList<CfgMessage> TryToLL1Parser(this CfgDocument cfg,out LL1Parser parser,IEnumerable<Token> tokenizer = null,IProgress<CfgLL1Progress> progress=null)
+		public static IList<CfgMessage> TryToLL1Parser(this CfgDocument cfg,out LL1Parser parser,ITokenizer tokenizer = null,IProgress<CfgLL1Progress> progress=null)
 		{
 			CfgLL1ParseTable parseTable;
 			var result = cfg.TryToLL1ParseTable(progress,out parseTable);

diff --git a/ll1/LL1DebugParser.cs b/ll1/LL1DebugParser.cs
@@ -13,6 +13,7 @@ public class LL1DebugParser : LL1Parser
 	{
 		CfgDocument _cfg;
 		CfgLL1ParseTable _parseTable;
+		ITokenizer _tokenizer;
 		IEnumerator<Token> _tokenEnum;
 		Token _errorToken;
 		Stack<string> _stack;
@@ -49,11 +50,21 @@ public override LLNodeType NodeType {
 				return LLNodeType.Initial;
 			}
 		}
-		public override void Restart(IEnumerable<Token> tokenizer)
+		public override void Restart(ITokenizer tokenizer)
 		{
 			Close();
+			_tokenizer = null;
 			if (null != tokenizer)
+			{
+				_tokenizer = tokenizer;
 				_tokenEnum = tokenizer.GetEnumerator();
+			}
+		}
+		public override void Restart(IEnumerable<char> input)
+		{
+			Close();
+			_tokenizer.Restart(input);
+			_tokenEnum = _tokenizer.GetEnumerator();
 		}
 		public override void Restart()
 		{
@@ -137,7 +148,7 @@ public override string Value {
 		/// <param name="tokenizer">The tokenizer to use </param>
 		/// <param name="startSymbol">The start symbol</param>
 		public LL1DebugParser(CfgDocument cfg,
-			IEnumerable<Token> tokenizer)
+			ITokenizer tokenizer)
 		{
 			_cfg = cfg;
 			_PopulateAttrs();

diff --git a/pck/ITokenizer.cs b/pck/ITokenizer.cs
@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Pck
+{
+	public interface ITokenizer : IEnumerable<Token>
+	{
+		void Restart(IEnumerable<char> input);
+
+	}
+}
diff --git a/pck/LL1Parser.cs b/pck/LL1Parser.cs
@@ -18,7 +18,8 @@ public abstract class LL1Parser : IDisposable
 		public abstract long Position { get; }
 		public abstract bool Read();
 		public abstract void Restart();
-		public abstract void Restart(IEnumerable<Token> tokenizer);
+		public abstract void Restart(ITokenizer tokenizer);
+		public abstract void Restart(IEnumerable<char> input);
 		public abstract void Close();
 		public abstract bool IsHidden { get; }
 		public abstract bool IsCollapsed { get; }

diff --git a/pck/LL1TableParser.cs b/pck/LL1TableParser.cs
@@ -12,6 +12,7 @@ namespace Pck
 	public class LL1TableParser : LL1Parser
 	{
 		int[][][] _parseTable;
+		ITokenizer _tokenizer;
 		IEnumerator<Token> _tokenEnum;
 		Token _errorToken;
 		Stack<int> _stack;
@@ -35,7 +36,7 @@ public override int SubstituteId {
 		}
 		public override KeyValuePair<string, object>[] GetAttributeSet(int symbolId)
 		{
-			if (0 < symbolId || _attributeSets.Length <= symbolId)
+			if (0 > symbolId || _attributeSets.Length <= symbolId)
 				return null;
 			return _attributeSets[symbolId];
 		}
@@ -82,11 +83,21 @@ public override LLNodeType NodeType {
 				return LLNodeType.Initial;
 			}
 		}
-		public override void Restart(IEnumerable<Token> tokenizer)
+		public override void Restart(ITokenizer tokenizer)
 		{
 			Close();
-			if(null!=tokenizer)
+			_tokenizer = null;
+			if (null != tokenizer)
+			{
+				_tokenizer = tokenizer;
 				_tokenEnum = tokenizer.GetEnumerator();
+			}
+		}
+		public override void Restart(IEnumerable<char> input)
+		{
+			Close();
+			_tokenizer.Restart(input);
+			_tokenEnum = _tokenizer.GetEnumerator();
 		}
 		public override void Restart()
 		{
@@ -161,7 +172,7 @@ public LL1TableParser(
 			int[] nodeFlags,
 			int[] substitutions,
 			KeyValuePair<string,object>[][] attributeSets,
-			IEnumerable<Token> tokenizer)
+			ITokenizer tokenizer)
 		{
 			_parseTable = parseTable;
 			_initCfg = initCfg;

diff --git a/pck/Lalr1Parser.cs b/pck/Lalr1Parser.cs
@@ -30,7 +30,8 @@ public string Rule {
 		public abstract KeyValuePair<string, object>[] GetAttributeSet(int symbolId);
 		public abstract object GetAttribute(string name, object @default = null);
 		public abstract bool Read();
-		public abstract void Restart(IEnumerable<Token> tokenizer);
+		public abstract void Restart(ITokenizer tokenizer);
+		public abstract void Restart(IEnumerable<char> input);
 		public abstract void Restart();
 		public abstract void Close();
 		void IDisposable.Dispose()

diff --git a/pck/Lalr1TableParser.cs b/pck/Lalr1TableParser.cs
@@ -12,6 +12,7 @@ public class Lalr1TableParser : Lalr1Parser
 		string[] _symbols;
 		int[] _nodeFlags;
 		int[] _substitutions;
+		ITokenizer _tokenizer;
 		IEnumerator<Token> _tokenEnum;
 		LRNodeType _nodeType;
 		Stack<int> _stack;
@@ -20,7 +21,7 @@ public class Lalr1TableParser : Lalr1Parser
 		int _errorId;
 		KeyValuePair<string, object>[][] _attributeSets;
 
-		public Lalr1TableParser(int[][][] parseTable, string[] symbols,int[] nodeFlags, int[] substitutions,KeyValuePair<string, object>[][] attributeSets,IEnumerable<Token> tokenizer)
+		public Lalr1TableParser(int[][][] parseTable, string[] symbols,int[] nodeFlags, int[] substitutions,KeyValuePair<string, object>[][] attributeSets,ITokenizer tokenizer)
 		{
 			_parseTable = parseTable;
 			_symbols = symbols;
@@ -34,7 +35,9 @@ public Lalr1TableParser(int[][][] parseTable, string[] symbols,int[] nodeFlags,
 				throw new ArgumentException("Error in symbol table", "symbols");
 
 			_stack = new Stack<int>();
-			_tokenEnum = tokenizer.GetEnumerator();
+			_tokenizer = tokenizer;
+			if(null!=tokenizer)
+				_tokenEnum = tokenizer.GetEnumerator();
 			_nodeType = LRNodeType.Initial;
 
 		}
@@ -174,7 +177,7 @@ public override bool Read()
 				_nodeType = LRNodeType.EndDocument;
 				return true;
 			}
-			else if (LRNodeType.EndDocument == _nodeType)
+			else if (LRNodeType.EndDocument == _nodeType || (LRNodeType.Error==_nodeType&&0==_stack.Count))
 				return false;
 			else if (_eosId == _tokenEnum.Current.SymbolId && LRNodeType.Error == _nodeType)
 				return false;
@@ -193,58 +196,65 @@ public override bool Read()
 					return true;
 				}
 			}
-			var entry = _parseTable[_stack.Peek()];
-			//(int RuleOrStateId, int Left, int[] Right) trns;
-			if(_errorId==_tokenEnum.Current.SymbolId)
-			{
-				_Panic();
-				return true;
-			}
-			int[] trns = entry[_tokenEnum.Current.SymbolId];
-			if (null==trns)
-			{
-				_Panic();
-				return true;
-			}
-			if (1== trns.Length) // shift or accept
+			if (0 < _stack.Count)
 			{
-				if (-1 != trns[0]) // shift
+				var entry = _parseTable[_stack.Peek()];
+				//(int RuleOrStateId, int Left, int[] Right) trns;
+				if (_errorId == _tokenEnum.Current.SymbolId)
 				{
-					_nodeType = LRNodeType.Shift;
-					_token = _tokenEnum.Current;
-					_tokenEnum.MoveNext();
-					_stack.Push(trns[0]);
+					_Panic();
 					return true;
 				}
-				else
-				{ // accept 
-				  //throw if _tok is not $ (end)
-					if (_eosId!= _tokenEnum.Current.SymbolId)
+				int[] trns = entry[_tokenEnum.Current.SymbolId];
+				if (null == trns)
+				{
+					_Panic();
+					return true;
+				}
+				if (1 == trns.Length) // shift or accept
+				{
+					if (-1 != trns[0]) // shift
 					{
-						_Panic();
+						_nodeType = LRNodeType.Shift;
+						_token = _tokenEnum.Current;
+						_tokenEnum.MoveNext();
+						_stack.Push(trns[0]);
 						return true;
 					}
+					else
+					{ // accept 
+					  //throw if _tok is not $ (end)
+						if (_eosId != _tokenEnum.Current.SymbolId)
+						{
+							_Panic();
+							return true;
+						}
 
-					_nodeType = LRNodeType.Accept;
-					_stack.Clear();
+						_nodeType = LRNodeType.Accept;
+						_stack.Clear();
+						return true;
+					}
+				}
+				else // reduce
+				{
+					_ruleDef = new int[trns.Length - 1];
+					for (var i = 1; i < trns.Length; i++)
+						_ruleDef[i - 1] = trns[i];
+					for (int i = 2; i < trns.Length; ++i)
+						_stack.Pop();
+
+					// There is a new number at the top of the stack. 
+					// This number is our temporary state. Get the symbol 
+					// from the left-hand side of the rule #. Treat it as 
+					// the next input token in the GOTO table (and place 
+					// the matching state at the top of the set stack).
+					_stack.Push(_parseTable[_stack.Peek()][trns[1]][0]);
+					_nodeType = LRNodeType.Reduce;
 					return true;
 				}
-			}
-			else // reduce
+			} else
 			{
-				_ruleDef = new int[trns.Length - 1];
-				for(var i=1;i<trns.Length;i++)
-					_ruleDef[i - 1] = trns[i];
-				for (int i = 2; i < trns.Length; ++i)
-					_stack.Pop();
-
-				// There is a new number at the top of the stack. 
-				// This number is our temporary state. Get the symbol 
-				// from the left-hand side of the rule #. Treat it as 
-				// the next input token in the GOTO table (and place 
-				// the matching state at the top of the set stack).
-				_stack.Push(_parseTable[_stack.Peek()][trns[1]][0]);
-				_nodeType = LRNodeType.Reduce;
+				_Panic();
 				return true;
 			}
 
@@ -265,18 +275,26 @@ public override void Restart()
 			_stack.Clear();
 			_nodeType = LRNodeType.Initial;
 		}
-		public override void Restart(IEnumerable<Token> tokenizer)
+		public override void Restart(ITokenizer tokenizer)
 		{
 			Close();
+			_tokenizer = null;
 			if (null != tokenizer)
+			{
+				_tokenizer = tokenizer;
 				_tokenEnum = tokenizer.GetEnumerator();
+			}
+		}
+		public override void Restart(IEnumerable<char> input)
+		{
+			Close();
+			_tokenizer.Restart(input);
+			_tokenEnum = _tokenizer.GetEnumerator();
 		}
 		void _Panic()
 		{
 			// This is primitive. Should see if the Dragon Book has something better
 			_nodeType = LRNodeType.Error;
-			var state = _stack.Peek();
-			var d = _parseTable[state];
 			int[] e;
 			_errorToken.Symbol = "#ERROR";
 			_errorToken.SymbolId = _errorId;
@@ -285,6 +303,10 @@ void _Panic()
 			_errorToken.Column = _tokenEnum.Current.Column;
 			_errorToken.Position = _tokenEnum.Current.Position;
 			var s = _tokenEnum.Current.SymbolId;
+			if (0 == _stack.Count)
+				return;
+			var state = _stack.Peek();
+			var d = _parseTable[state];
 			if (_errorId!=s && null!=(e=d[s]) && _eosId != s)
 			{
 				_errorToken.Value += _tokenEnum.Current.Value;