implab/ImplabNet Commit - r176:0c3c69fe225b

rewritten the text scanner

cin -

r176:0c3c69fe225b ref20160224

parent child

Context file:

r176:0c3c69fe225b

Collapse all files

Implab/Formats/ReaderScanner.cs created 644 +30 0

@@ -0,0 +1,30
	1	using System;
	2	using System.IO;
	3
	4	namespace Implab.Formats {
	5	public class ReaderScanner: TextScanner {
	6	const int CHUNK_SIZE = 1024;
	7	const int BUFFER_MAX = CHUNK_SIZE*1024;
	8
	9	readonly TextReader m_reader;
	10
	11	public ReaderScanner(TextReader reader, int limit, int chunk) : base(limit, chunk) {
	12	Safe.ArgumentNotNull(reader, "reader");
	13	m_reader = reader;
	14	}
	15
	16	public ReaderScanner(TextReader reader) : this(reader, BUFFER_MAX, CHUNK_SIZE) {
	17	}
	18
	19	protected override int Read(char[] buffer, int offset, int size) {
	20	return m_reader.Read(buffer, offset, size);
	21	}
	22
	23	protected override void Dispose(bool disposing) {
	24	if (disposing)
	25	Safe.Dispose(m_reader);
	26	base.Dispose(disposing);
	27	}
	28	}
	29	}
	30

Implab/Formats/ScannerContext.cs created 644 +24 0

@@ -0,0 +1,24
	1	using System;
	2
	3	namespace Implab.Formats {
	4	public class ScannerContext<TTag> {
	5	public int[,] Dfa { get; private set; }
	6	public bool[] Final { get; private set; }
	7	public TTag[][] Tags { get; private set; }
	8	public int State { get; private set; }
	9	public int[] Alphabet { get; private set; }
	10
	11	public ScannerContext(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet) {
	12	Dfa = dfa;
	13	Final = final;
	14	Tags = tags;
	15	State = state;
	16	Alphabet = alphabet;
	17	}
	18
	19	public bool Execute(TextScanner scanner, out TTag[] tag) {
	20	return scanner.ReadToken(Dfa, Final, Tags, State, Alphabet, out tag);
	21	}
	22	}
	23	}
	24

Implab/Formats/StringScanner.cs created 644 +26 0

@@ -0,0 +1,26
	1	using System;
	2
	3	namespace Implab.Formats {
	4	public class StringScanner: TextScanner {
	5	const int CHUNK_SIZE = 1024;
	6
	7	readonly string m_text;
	8	int m_pos;
	9
	10	public StringScanner(string text) : base(text.Length, text.Length < CHUNK_SIZE ? text.Length : CHUNK_SIZE) {
	11	m_text = text;
	12	Feed();
	13	}
	14
	15	protected override int Read(char[] buffer, int offset, int size) {
	16	var actual = size + m_pos > m_text.Length ? m_text.Length - m_pos : size;
	17
	18	m_text.CopyTo(m_pos,buffer,offset, actual);
	19
	20	m_pos += actual;
	21
	22	return actual;
	23	}
	24	}
	25	}
	26

Implab/Automaton/DFATable.cs +16 -8

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton {
                 public class DFATable : IDFATableBuilder {
                     int m_stateCount;
                     int m_symbolCount;
                     int m_initialState;
                     readonly HashSet<int> m_finalStates = new HashSet<int>();
                     readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
                     #region IDFADefinition implementation
                     public bool IsFinalState(int s) {
                         Safe.ArgumentInRange(s, 0, m_stateCount, "s");
                         return m_finalStates.Contains(s);
                     }
                     public IEnumerable<int> FinalStates {
                         get {
                             return m_finalStates;
                         }
                     }
                     public int StateCount {
                         get { return m_stateCount; }
                     }
                     public int AlphabetSize {
                         get { return m_symbolCount; }
                     }
                     public int InitialState {
                         get { return m_initialState; }
                     }
                     #endregion
                     public void SetInitialState(int s) {
                         Safe.ArgumentAssert(s >= 0, "s");
                         m_initialState = s;
                     }
                     public void MarkFinalState(int state) {
                         m_finalStates.Add(state);
                     }
                     public void Add(AutomatonTransition item) {
                         Safe.ArgumentAssert(item.s1 >= 0, "item");
                         Safe.ArgumentAssert(item.s2 >= 0, "item");
                         Safe.ArgumentAssert(item.edge >= 0, "item");
                         m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
                         m_symbolCount = Math.Max(m_symbolCount, item.edge);
                         m_transitions.Add(item);
                     }
                     public void Clear() {
                         m_stateCount = 0;
                         m_symbolCount = 0;
                         m_finalStates.Clear();
                         m_transitions.Clear();
                     }
                     public bool Contains(AutomatonTransition item) {
                         return m_transitions.Contains(item);
                     }
                     public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
                         m_transitions.CopyTo(array, arrayIndex);
                     }
                     public bool Remove(AutomatonTransition item) {
                         m_transitions.Remove(item);
                     }
                     public int Count {
                         get {
                             return m_transitions.Count;
                         }
                     }
                     public bool IsReadOnly {
                         get {
                             return false;
                         }
                     }
                     public IEnumerator<AutomatonTransition> GetEnumerator() {
                         return m_transitions.GetEnumerator();
                     }
                     System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
                         return GetEnumerator();
                     }
-                    public DFAStateDescriptor[] CreateTransitionTable() {
+                    public int[,] CreateTransitionTable() {
-                        var table = new DFAStateDescriptor[StateCount];
+                        var table = new int[StateCount,AlphabetSize];
+                        for (int i = 0; i < StateCount; i++)
+                            for (int j = 0; i < AlphabetSize; j++)
+                                table[i, j] = DFAConst.UNREACHABLE_STATE;
-                        foreach (var t in this) {
+                        foreach (var t in this)
-                            if (table[t.s1].transitions == null)
+                            table[t.s1,t.edge] = t.s2;
-                                table[t.s1] = new DFAStateDescriptor(AlphabetSize, IsFinalState(t.s1));
-                            if (table[t.s2].transitions == null)
+                        return table;
-                                table[t.s2] = new DFAStateDescriptor(AlphabetSize, IsFinalState(t.s2));
-                            table[t.s1].transitions[t.edge] = t.s2;
                     }
+                    public bool[] CreateFinalStateTable() {
+                        var table = new bool[StateCount];
+                        foreach (var s in FinalStates)
+                            table[s] = true;
                         return table;
                     }
                     /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
                     /// <remarks>
                     /// В процессе построения минимального автомата требуется разделить множество состояний,
                     /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
                     /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
                     /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
                     /// </remarks>
                     /// <returns>The final states.</returns>
                     protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
                         return new HashSet<int>[] { m_finalStates };
                     }
                     protected void Optimize(
                         IDFATableBuilder optimalDFA,
                         IDictionary<int,int> alphabetMap,
                         IDictionary<int,int> stateMap
                     ) {
                         Safe.ArgumentNotNull(optimalDFA, "dfa");
                         Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
                         Safe.ArgumentNotNull(stateMap, "stateMap");
                         var setComparer = new CustomEqualityComparer<HashSet<int>>(
                             (x, y) => x.SetEquals(y),
                             s => s.Sum(x => x.GetHashCode())
                         );
                         var optimalStates = new HashSet<HashSet<int>>(setComparer);
                         var queue = new HashSet<HashSet<int>>(setComparer);
                         // получаем конечные состояния, сгруппированные по маркерам
                         optimalStates.UnionWith(
                             GroupFinalStates()
                         );
                         var state = new HashSet<int>(
                             Enumerable
                             .Range(0, m_stateCount - 1)
                             .Where(i => !m_finalStates.Contains(i))
                         );
                         optimalStates.Add(state);
                         queue.Add(state);
                         var rmap = m_transitions
                             .GroupBy(t => t.s2)
                             .ToLookup(
                                 g => g.Key, // s2
                                 g => g.ToLookup(t => t.edge, t => t.s1)
                             );
                         while (queue.Count > 0) {
                             var stateA = queue.First();
                             queue.Remove(stateA);
                             for (int c = 0; c < m_symbolCount; c++) {
                                 var stateX = new HashSet<int>();
                                 foreach(var a in stateA)
                                     stateX.UnionWith(rmap[a][c]); // all states from wich 'c' leads to 'a'
                                 foreach (var stateY in optimalStates.ToArray()) {
                                     if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
                                         var stateR1 = new HashSet<int>(stateY);
                                         var stateR2 = new HashSet<int>(stateY);
                                         stateR1.IntersectWith(stateX);
                                         stateR2.ExceptWith(stateX);
                                         optimalStates.Remove(stateY);
                                         optimalStates.Add(stateR1);
                                         optimalStates.Add(stateR2);
                                         if (queue.Contains(stateY)) {
                                             queue.Remove(stateY);
                                             queue.Add(stateR1);
                                             queue.Add(stateR2);
                                         } else {
                                             queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
                                         }
                                     }
                                 }
                             }
                         }
                         // карта получения оптимального состояния по соотвествующему ему простому состоянию
                         var nextState = 0;
                         foreach (var item in optimalStates) {
                             var id = nextState++;
                             foreach (var s in item)
                                 stateMap[s] = id;
                         }
                         // получаем минимальный алфавит
                         // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
                         // для этого используем алгоритм кластеризации, сначала
                         // считаем, что все символы не различимы
                         var minClasses = new HashSet<HashSet<int>>(setComparer);
                         var alphaQueue = new Queue<HashSet<int>>();
                         alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
                         // для всех состояний, будем проверять каждый класс на различимость,
                         // т.е. символы различимы, если они приводят к разным состояниям
                         for (int s = 0 ; s < optimalStates.Count; s++) {
                             var newQueue = new Queue<HashSet<int>>();
                             foreach (var A in alphaQueue) {
                                 // классы из одного символа делить бесполезно, переводим их сразу в
                                 // результирующий алфавит
                                 if (A.Count == 1) {
                                     minClasses.Add(A);
                                     continue;
                                 }
                                 // различаем классы символов, которые переводят в различные оптимальные состояния
                                 // optimalState -> alphaClass
                                 var classes = new Dictionary<int, HashSet<int>>();
                                 foreach (var term in A) {
                                     // ищем все переходы класса по символу term
                                     var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray();
                                     var s2 = res.Length > 0 ? res[0] : -1;
                                     HashSet<int> a2;
                                     if (!classes.TryGetValue(s2, out a2)) {
                                         a2 = new HashSet<int>();
                                         newQueue.Enqueue(a2);
                                         classes[s2] = a2;
                                     }
                                     a2.Add(term);
                                 }
                             }
                             if (newQueue.Count == 0)
                                 break;
                             alphaQueue = newQueue;
                         }
                         // после окончания работы алгоритма в очереди останутся минимальные различимые классы
                         // входных символов
                         foreach (var A in alphaQueue)
                             minClasses.Add(A);
                         // построение отображения алфавитов входных символов.
                         // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
                         // специальное значение, тогда сохраним минимальный класс,
                         // содержащий этот символ на томже месте.
                         var nextCls = 0;
                         foreach (var item in minClasses) {
                             if (nextCls == DFAConst.UNCLASSIFIED_INPUT)
                                 nextCls++;
                             // сохраняем DFAConst.UNCLASSIFIED_INPUT
                             var cls = item.Contains(DFAConst.UNCLASSIFIED_INPUT) ? DFAConst.UNCLASSIFIED_INPUT : nextCls;
                             foreach (var a in item)
                                 alphabetMap[a] = cls;
                             nextCls++;
                         }
                         // построение автомата
                         optimalDFA.SetInitialState(stateMap[m_initialState]);
                         foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
                             optimalDFA.MarkFinalState(sf);
                         foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
                             optimalDFA.Add(t);
                     }
                     protected void PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
                         Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
                         Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
                         foreach(var t in m_transitions)
                             Console.WriteLine(
                                 "[{0}] -{{{1}}}-> [{2}]{3}",
                                 String.Join(",", stateAlphabet.GetSymbols(t.s1)),
                                 String.Join("", inputAlphabet.GetSymbols(t.edge)),
                                 String.Join(",", stateAlphabet.GetSymbols(t.s2)),
                                 m_finalStates.Contains(t.s2) ? "$" : ""
                             );
                     }
                 }
             }

Implab/Automaton/IndexedAlphabetBase.cs +22 -66

@@ -1,94 +1,50
1	using Implab;	1	using Implab;
2	using System;	2	using System;
3	using System.Collections.Generic;	3	using System.Collections.Generic;
4	using System.Diagnostics;	4	using System.Diagnostics;
5	using System.Linq;	5	using System.Linq;
6		6
7	namespace Implab.Automaton {	7	namespace Implab.Automaton {
8	/// <summary>	8	/// <summary>
9	/// Indexed alphabet is the finite set of symbols where each symbol has a zero-based unique index.	9	/// Indexed alphabet is the finite set of symbols where each symbol has a zero-based unique index.
10	/// </summary>	10	/// </summary>
11	/// <remarks>	11	/// <remarks>
12	/// Indexed alphabets are usefull in bulting efficient translations from source alphabet	12	/// Indexed alphabets are usefull in bulting efficient translations from source alphabet
13	/// to the input alphabet of the automaton. It's assumed that the index to the symbol match	13	/// to the input alphabet of the automaton. It's assumed that the index to the symbol match
14	/// is well known and documented.	14	/// is well known and documented.
15	/// </remarks>	15	/// </remarks>
16	public abstract class IndexedAlphabetBase<T> : IAlphabet~~Builder~~<T> {	16	public abstract class IndexedAlphabetBase<T> : MapAlphabet<T> {
17	int m_nextId = 1;
18	readonly int[] m_map;
19
20	protected IndexedAlphabetBase(int mapSize) {
21	m_map = new int[mapSize];
22	}
23
24	protected IndexedAlphabetBase(int[] map) {
25	Debug.Assert(map != null && map.Length > 0);
26	Debug.Assert(map.All(x => x >= 0));
27
28	m_map = map;
29	m_nextId = map.Max() + 1;
30	}
31
32	public int DefineSymbol(T symbol) {
33	var index = GetSymbolIndex(symbol);
34	if (m_map[index] == DFAConst.UNCLASSIFIED_INPUT)
35	m_map[index] = m_nextId++;
36	return m_map[index];
37	}
38
39	public int DefineSymbol(T symbol, int cls) {
40	var index = GetSymbolIndex(symbol);
41	m_map[index] = cls;
42	m_nextId = Math.Max(cls + 1, m_nextId);
43	return cls;
44	}
45		17
46	public int DefineClass(IEnumerable<T> symbols) {	18	protected IndexedAlphabetBase() :base(true, null) {
47	return DefineClass(symbols, m_nextId);
48	}
49
50	public int DefineClass(IEnumerable<T> symbols, int cls) {
51	Safe.ArgumentNotNull(symbols, "symbols");
52	symbols = symbols.Distinct();
53
54	foreach (var symbol in symbols)
55	m_map[GetSymbolIndex(symbol)] = cls;
56
57	m_nextId = Math.Max(cls + 1, m_nextId);
58
59	return cls;
60	}
61
62	public virtual int Translate(T symbol) {
63	return m_map[GetSymbolIndex(symbol)];
64	}
65
66	public int Count {
67	get { return m_nextId; }
68	}
69
70	public bool Contains(T symbol) {
71	return true;
72	}
73
74	public IEnumerable<T> GetSymbols(int cls) {
75	for (var i = 0; i < m_map.Length; i++)
76	if (m_map[i] == cls)
77	yield return GetSymbolByIndex(i);
78	}	19	}
79		20
80	public abstract int GetSymbolIndex(T symbol);	21	public abstract int GetSymbolIndex(T symbol);
81		22
82	public abstract T GetSymbolByIndex(int index);
83
84	public abstract IEnumerable<T> InputSymbols { get; }
85
86	/// <summary>	23	/// <summary>
87	/// Gets the translation map from the index of the symbol to it's class this is usefull for the optimized input symbols transtaion.	24	/// Gets the translation map from the index of the symbol to it's class this is usefull for the optimized input symbols transtaion.
88	/// </summary>	25	/// </summary>
		26	/// <remarks>
		27	/// The map is continous and start from the symbol with zero code. The last symbol
		28	/// in the map is the last classified symbol in the alphabet, i.e. the map can be
		29	/// shorter then the whole alphabet.
		30	/// </remarks>
89	/// <returns>The translation map.</returns>	31	/// <returns>The translation map.</returns>
90	public int[] GetTranslationMap() {	32	public int[] GetTranslationMap() {
91	return m_map;	33	Dictionary<int,int> map = new Dictionary<int, int>();
		34
		35	int max;
		36	foreach (var p in Mappings) {
		37	var index = GetSymbolIndex(p.Key);
		38	max = Math.Max(max, index);
		39	map[index] = p.Value;
		40	}
		41
		42	var result = new int[max + 1];
		43
		44	for (int i = 0; i < result.Length; i++)
		45	map.TryGetValue(i, out result[i]);
		46
		47	return result;
92	}	48	}
93	}	49	}
94	}	50	}

Implab/Automaton/MapAlphabet.cs +7 0

             using System;
             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton {
                 public class MapAlphabet<T> : IAlphabetBuilder<T> {
                     readonly Dictionary<T,int> m_map;
                     int m_nextCls;
                     readonly bool m_supportUnclassified;
                     public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
                         m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
                         m_supportUnclassified = supportUnclassified;
                         m_nextCls = supportUnclassified ? 1 : 0;
                     }
                     #region IAlphabetBuilder implementation
                     public int DefineSymbol(T symbol) {
                         int cls;
                         return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
                     }
                     public int DefineSymbol(T symbol, int cls) {
                         Safe.ArgumentAssert(cls >= 0, "cls");
                         m_nextCls = Math.Max(cls + 1, m_nextCls);
                         m_map.Add(symbol, cls);
                         return cls;
                     }
                     public int DefineClass(IEnumerable<T> symbols) {
                         return DefineClass(symbols, m_nextCls);
                     }
                     public int DefineClass(IEnumerable<T> symbols, int cls) {
                         Safe.ArgumentAssert(cls >= 0, "cls");
                         Safe.ArgumentNotNull(symbols, "symbols");
                         m_nextCls = Math.Max(cls + 1, m_nextCls);
                         foreach (var symbol in symbols)
                             m_map[symbol] = cls;
                         return cls;
                     }
                     #endregion
                     #region IAlphabet implementation
                     public int Translate(T symbol) {
                         int cls;
                         if (m_map.TryGetValue(symbol, out cls))
                             return cls;
                         if (!m_supportUnclassified)
                             throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
                         return DFAConst.UNCLASSIFIED_INPUT;
                     }
                     public int Count {
                         get {
                             return m_nextCls;
                         }
                     }
                     public bool Contains(T symbol) {
                         return m_supportUnclassified || m_map.ContainsKey(symbol);
                     }
                     public IEnumerable<T> GetSymbols(int cls) {
+                        Safe.ArgumentAssert(cls > 0, "cls");
                         return m_map.Where(p => p.Value == cls).Select(p => p.Key);
                     }
                     #endregion
+                    public IEnumerable<KeyValuePair<T,int>> Mappings {
+                        get {
+                            return m_map;
+                        }
+                    }
                 }
             }

Implab/Automaton/RegularExpressions/Grammar.cs +12 -3

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton.RegularExpressions {
                 /// <summary>
                 /// Базовый абстрактный класс. Грамматика, позволяет формулировать выражения над алфавитом типа <c>char</c>.
                 /// </summary>
                 public abstract class Grammar<TSymbol, TTag> {
                     protected abstract IAlphabetBuilder<TSymbol> AlphabetBuilder {
                         get;
                     }
                     protected SymbolToken<TTag> UnclassifiedToken() {
                         return new SymbolToken<TTag>(DFAConst.UNCLASSIFIED_INPUT);
                     }
                     protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "alphabet");
                         foreach (var ch in alphabet)
                             AlphabetBuilder.DefineSymbol(ch);
                     }
                     protected Token<TTag> SymbolToken(TSymbol symbol) {
                         return Token<TTag>.New(TranslateOrAdd(symbol));
                     }
                     protected Token<TTag> SymbolToken(IEnumerable<TSymbol> symbols) {
                         Safe.ArgumentNotNull(symbols, "symbols");
                         return Token<TTag>.New(TranslateOrAdd(symbols).ToArray());
                     }
                     protected Token<TTag> SymbolSetToken(params TSymbol[] set) {
                         return SymbolToken(set);
                     }
                     int TranslateOrAdd(TSymbol ch) {
                         var t = AlphabetBuilder.Translate(ch);
                         if (t == DFAConst.UNCLASSIFIED_INPUT)
                             t = AlphabetBuilder.DefineSymbol(ch);
                         return t;
                     }
                     IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
                         return symbols.Distinct().Select(TranslateOrAdd);
                     }
                     int TranslateOrDie(TSymbol ch) {
                         var t = AlphabetBuilder.Translate(ch);
                         if (t == DFAConst.UNCLASSIFIED_INPUT)
                                 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
                         return t;
                     }
                     IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
                         return symbols.Distinct().Select(TranslateOrDie);
                     }
                     protected Token<TTag> SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
                         Safe.ArgumentNotNull(symbols, "symbols");
                         return Token<TTag>.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
                     }
-                    protected abstract IAlphabetBuilder<TSymbol> CreateAlphabet();
+                    protected abstract IndexedAlphabetBase<TSymbol> CreateAlphabet();
-                    protected RegularDFA<TSymbol, TTag> BuildDFA(Token<TTag> regexp) {
+                    protected ScannerContext<TTag> BuildScannerContext(Token<TTag> regexp) {
                         var dfa = new RegularDFA<TSymbol, TTag>(AlphabetBuilder);
                         var visitor = new RegularExpressionVisitor<TTag>();
                         regexp.Accept( visitor );
                         visitor.BuildDFA(dfa);
                         if (dfa.IsFinalState(dfa.InitialState))
                             throw new ApplicationException("The specified language contains empty token");
-                        return dfa.Optimize(CreateAlphabet());
+                        var ab = CreateAlphabet();
+                        var optimal = dfa.Optimize(ab);
+                        return new ScannerContext<TTag>(
+                            optimal.CreateTransitionTable(),
+                            optimal.CreateFinalStateTable(),
+                            optimal.CreateTagTable(),
+                            optimal.InitialState,
+                            ab.GetTranslationMap()
+                        );
                     }
                 }
             }

Implab/Automaton/RegularExpressions/RegularDFA.cs +4 -9

             using System;
             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton.RegularExpressions {
                 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
                     readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
                     readonly IAlphabet<TInput> m_alphabet;
                     public RegularDFA(IAlphabet<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "aplhabet");
                         m_alphabet = alphabet;
                     }
                     public IAlphabet<TInput> InputAlphabet {
                         get {
                             return m_alphabet;
                         }
                     }
                     public void MarkFinalState(int s, TTag[] tags) {
                         MarkFinalState(s);
                         SetStateTag(s, tags);
                     }
                     public void SetStateTag(int s, TTag[] tags) {
                         Safe.ArgumentNotNull(tags, "tags");
                         m_tags[s] = tags;
                     }
                     public TTag[] GetStateTag(int s) {
                         TTag[] tags;
                         return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
                     }
-                    public new DFAStateDescriptor<TTag>[] CreateTransitionTable() {
+                    public TTag[][] CreateTagTable() {
-                        var table = new DFAStateDescriptor<TTag>[StateCount];
+                        var table = new TTag[StateCount][];
-                        foreach (var t in this) {
+                        foreach (var pair in m_tags)
-                            if (table[t.s1].transitions == null)
+                            table[pair.Key] = pair.Value;
-                                table[t.s1] = new DFAStateDescriptor<TTag>(AlphabetSize, IsFinalState(t.s1), GetStateTag(t.s1));
-                            if (table[t.s2].transitions == null)
-                                table[t.s2] = new DFAStateDescriptor<TTag>(AlphabetSize, IsFinalState(t.s2), GetStateTag(t.s2));
-                            table[t.s1].transitions[t.edge] = t.s2;
                         return table;
                     }
                     /// <summary>
                     /// Optimize the specified alphabet.
                     /// </summary>
                     /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
                     public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "alphabet");
                         var dfa = new RegularDFA<TInput, TTag>(alphabet);
                         var states = new DummyAlphabet(StateCount);
                         var alphaMap = new Dictionary<int,int>();
                         var stateMap = new Dictionary<int,int>();
                         Optimize(dfa, alphaMap, stateMap);
                         // mark tags in the new DFA
                         foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
                             dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
                         // make the alphabet for the new DFA
                         foreach (var pair in alphaMap)
                             alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
                         return dfa;
                     }
                     protected override IEnumerable<HashSet<int>> GroupFinalStates() {
                         var arrayComparer = new CustomEqualityComparer<TTag[]>(
                             (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
                             x => x.Sum(it => x.GetHashCode())
                         );
                         return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
                     }
                 }
             }

Implab/Formats/ByteAlphabet.cs +1 -5

             using System.Collections.Generic;
             using System.Linq;
             using Implab.Automaton;
             namespace Implab.Formats {
                 public class ByteAlphabet : IndexedAlphabetBase<byte> {
-                    public ByteAlphabet() : base(byte.MaxValue + 1){
+                    public ByteAlphabet() {
                     }
                     #region implemented abstract members of IndexedAlphabetBase
                     public override int GetSymbolIndex(byte symbol) {
                         return (int)symbol;
                     }
-                    public override byte GetSymbolByIndex(int index) {
-                        return (byte)index;
                     public IEnumerable<byte> InputSymbols {
                         get {
                             return Enumerable.Range(byte.MinValue, byte.MaxValue).Cast<byte>();
                         }
                     }
                     #endregion
                 }
             }

Implab/Formats/CharAlphabet.cs +2 -7

             using System.Collections.Generic;
             using System.Linq;
             using Implab.Automaton;
             namespace Implab.Formats {
                 public class CharAlphabet: IndexedAlphabetBase<char> {
-                    public CharAlphabet()
+                    public CharAlphabet() {
-                        : base(char.MaxValue + 1) {
                     }
                     public override int GetSymbolIndex(char symbol) {
                         return symbol;
                     }
-                    public override char GetSymbolByIndex(int index) {
+                    public IEnumerable<char> InputSymbols {
-                        return (char)index;
-                    public override IEnumerable<char> InputSymbols {
                         get { return Enumerable.Range(char.MinValue, char.MaxValue).Cast<char>(); }
                     }
                 }
             }

Implab/Formats/JSON/JSONGrammar.cs +7 -14

             using System.Linq;
             using Implab.Automaton.RegularExpressions;
             using System;
             using Implab.Automaton;
             namespace Implab.Formats.JSON {
                 class JSONGrammar : Grammar<char,JSONGrammar.TokenType> {
                     public enum TokenType {
                         None,
                         BeginObject,
                         EndObject,
                         BeginArray,
                         EndArray,
                         String,
                         Number,
                         Literal,
                         NameSeparator,
                         ValueSeparator,
                         StringBound,
                         EscapedChar,
                         UnescapedChar,
-                        EscapedUnicode,
+                        EscapedUnicode
-                        Minus,
-                        Plus,
-                        Sign,
-                        Integer,
-                        Dot,
-                        Exp
                     }
                     static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();
                     public static JSONGrammar Instance {
                         get { return _instance.Value; }
                     }
-                    readonly RegularDFA<char, TokenType> m_jsonDFA;
+                    readonly ScannerContext<TokenType> m_jsonDFA;
-                    readonly RegularDFA<char, TokenType> m_stringDFA;
+                    readonly ScannerContext<TokenType> m_stringDFA;
                     public JSONGrammar() {
                         DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
                         var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
                         var digit9 = SymbolRangeToken('1', '9');
                         var zero = SymbolToken('0');
                         var digit = zero.Or(digit9);
                         var dot = SymbolToken('.');
                         var minus = SymbolToken('-');
                         var sign = SymbolSetToken('-', '+');
                         var expSign = SymbolSetToken('e', 'E');
                         var letters = SymbolRangeToken('a', 'z');
                         var integer = zero.Or(digit9.Cat(digit.EClosure()));
                         var frac = dot.Cat(digit.Closure());
                         var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
                         var quote = SymbolToken('"');
                         var backSlash = SymbolToken('\\');
                         var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
                         var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
                         var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
                         var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
                         var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
                         var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
                         var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
                         var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
                         var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
                         var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
                         var literal = letters.Closure();
                         var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
                         var jsonExpression =
                             number.Tag(TokenType.Number)
                             .Or(literal.Tag(TokenType.Literal))
                             .Or(quote.Tag(TokenType.StringBound))
                             .Or(beginObject.Tag(TokenType.BeginObject))
                             .Or(endObject.Tag(TokenType.EndObject))
                             .Or(beginArray.Tag(TokenType.BeginArray))
                             .Or(endArray.Tag(TokenType.EndArray))
                             .Or(nameSep.Tag(TokenType.NameSeparator))
                             .Or(valueSep.Tag(TokenType.ValueSeparator));
                         var jsonStringExpression =
                             quote.Tag(TokenType.StringBound)
                             .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
                             .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
                             .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
-                        m_jsonDFA = BuildDFA(jsonExpression);
+                        m_jsonDFA = BuildScannerContext(jsonExpression);
-                        m_stringDFA = BuildDFA(jsonStringExpression);
+                        m_stringDFA = BuildScannerContext(jsonStringExpression);
                     }
-                    public RegularDFA<char, TokenType> JsonDFA {
+                    public ScannerContext<TokenType> JsonDFA {
                         get {
                             return m_jsonDFA;
                         }
                     }
-                    public RegularDFA<char,TokenType> JsonStringDFA {
+                    public ScannerContext<TokenType> JsonStringDFA {
                         get {
                             return m_stringDFA;
                         }
                     }
                     Token<TokenType> SymbolRangeToken(char start, char stop) {
                         return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
                     }
                     protected override IAlphabetBuilder<char> CreateAlphabet() {
                         return new CharAlphabet();
                     }
                 }
             }

Implab/Formats/JSON/JSONScanner.cs +43 -38

             using System;
             using System.Globalization;
             using Implab.Automaton;
+            using System.Text;
+            using Implab.Components;
+            using System.IO;
+            using Implab.Automaton.RegularExpressions;
             namespace Implab.Formats.JSON {
                 /// <summary>
                 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
                 /// </summary>
-                public class JSONScanner : Scanner<object> {
+                public class JSONScanner : Disposable {
-                    char[] m_stringBuffer;
+                    readonly StringBuilder m_builder = new StringBuilder();
-                    DFAStateDescriptior<>[] m_stringDFA;
-                    int[] m_stringAlphabet;
+                    readonly ScannerContext<JSONGrammar.TokenType> m_jsonScanner = JSONGrammar.Instance.JsonDFA;
+                    readonly ScannerContext<JSONGrammar.TokenType> m_stringScanner = JSONGrammar.Instance.JsonStringDFA;
+                    readonly TextScanner m_scanner;
                     /// <summary>
                     /// Создает новый экземпляр сканнера
                     /// </summary>
-                    public JSONScanner()
+                    public JSONScanner(string text) {
-                        : base(JSONGrammar.Instance.JsonDFA.GetTransitionTable(), JSONGrammar.Instance.JsonDFA.Alphabet.GetTranslationMap()) {
+                        Safe.ArgumentNotEmpty(text, "text");
-                        m_stringBuffer = new char[1024];
-                        var dfa = JSONGrammar.Instance.JsonStringDFA;
+                        m_scanner = new StringScanner(text);
-                        m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
-                        m_stringDFA = dfa.States;
+                    public JSONScanner(TextReader reader, int bufferMax, int chunkSize) {
+                        Safe.ArgumentNotNull(reader, "reader");
+                        m_scanner = new ReaderScanner(reader);
                     }
                     /// <summary>
                     /// Читает следующий лексический элемент из входных данных.
                     /// </summary>
                     /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
                     /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
                     /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
                     /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
                     /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
                     public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
-                        if (ReadTokenInternal()) {
+                        JSONGrammar.TokenType[] tag;
-                            switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
+                        if (m_jsonScanner.Execute(m_scanner, out tag)) {
+                            switch (tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
                                     tokenValue = ReadString();
                                     tokenType = JsonTokenType.String;
                                     break;
                                 case JSONGrammar.TokenType.Number:
-                                    tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
+                                    tokenValue = Double.Parse(m_scanner.GetTokenValue(), CultureInfo.InvariantCulture);
                                     tokenType = JsonTokenType.Number;
                                     break;
                                 default:
-                                    tokenType = (JsonTokenType)m_currentState.tag[0];
+                                    tokenType = (JsonTokenType)tag[0];
-                                    tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
+                                    tokenValue = m_scanner.GetTokenValue();
                                     break;
                             }
                             return true;
                         }
                         tokenValue = null;
                         tokenType = JsonTokenType.None;
                         return false;
                     }
                     string ReadString() {
                         int pos = 0;
-                        Switch(m_stringDFA, m_stringAlphabet);
+                        char[] buf = new char[6]; // the buffer for unescaping chars
-                        while (ReadTokenInternal()) {
-                            switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
+                        JSONGrammar.TokenType[] tag;
+                        m_builder.Clear();
+                        while (m_stringScanner.Execute(m_scanner, out tag)) {
+                            switch (tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
-                                    Restore();
+                                    return m_builder.ToString();
-                                    return new String(m_stringBuffer, 0, pos);
                                 case JSONGrammar.TokenType.UnescapedChar:
-                                    EnsureStringBufferSize(pos + m_tokenLen);
+                                    m_scanner.CopyTokenTo(m_builder);
-                                    Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
-                                    pos += m_tokenLen;
                                     break;
-                                case JSONGrammar.TokenType.EscapedUnicode:
+                                case JSONGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
-                                    EnsureStringBufferSize(pos + 1);
+                                    m_scanner.CopyTokenTo(buf, 0);
-                                    m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
+                                    m_builder.Append(StringTranslator.TranslateHexUnicode(buf, 2));
                                     pos++;
                                     break;
-                                case JSONGrammar.TokenType.EscapedChar:
+                                case JSONGrammar.TokenType.EscapedChar:  // \t - escape sequence
-                                    EnsureStringBufferSize(pos + 1);
+                                    m_scanner.CopyTokenTo(buf, 0);
-                                    m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
+                                    m_builder.Append(StringTranslator.TranslateEscapedChar(buf[1]));
-                                    pos++;
                                     break;
                                 default:
                                     break;
                             }
                         }
                         throw new ParserException("Unexpected end of data");
                     }
-                    void EnsureStringBufferSize(int size) {
-                        if (size > m_stringBuffer.Length) {
-                            var newBuffer = new char[size];
-                            m_stringBuffer.CopyTo(newBuffer, 0);
-                            m_stringBuffer = newBuffer;
                 }
             }

Implab/Formats/JSON/StringTranslator.cs +5 -6

             using Implab;
-            using Implab.Parsing;
+            using Implab.Formats;
             using System;
             using System.Collections.Generic;
             using System.Diagnostics;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
-            namespace Implab.JSON {
+            namespace Implab.Formats.JSON {
                 /// <summary>
                 /// Класс для преобразования экранированной строки JSON
                 /// </summary>
-                public class StringTranslator : Scanner {
+                public class StringTranslator : TextScanner<JSONGrammar.TokenType> {
                     static readonly char[] _escMap;
                     static readonly int[] _hexMap;
                     static StringTranslator() {
                         var chars = new char[] { 'b', 'f', 't', 'r', 'n', '\\', '/' };
                         var vals = new char[] { '\b', '\f', '\t', '\r', '\n', '\\', '/' };
                         _escMap = new char[chars.Max() + 1];
                         for (int i = 0; i < chars.Length; i++)
                             _escMap[chars[i]] = vals[i];
                         var hexs = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' };
                         var ints = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 10, 11, 12, 13, 14, 15 };
                         _hexMap = new int[hexs.Max() + 1];
                         for (int i = 0; i < hexs.Length; i++)
                             _hexMap[hexs[i]] = ints[i];
                     }
-                    public StringTranslator()
+                    public StringTranslator() {
-                        : base(JSONGrammar.Instance.JsonStringDFA.States, JSONGrammar.Instance.JsonStringDFA.Alphabet.GetTranslationMap()) {
                     }
                     public string Translate(string data) {
                         Safe.ArgumentNotNull(data, "data");
                         return Translate(data.ToCharArray());
                     }
                     public string Translate(char[] data) {
                         Safe.ArgumentNotNull(data, "data");
                         return Translate(data, data.Length);
                     }
                     public string Translate(char[] data, int length) {
                         Safe.ArgumentNotNull(data, "data");
                         Safe.ArgumentInRange(length, 0, data.Length, "length");
                         var translated = new char[length];
                         Feed(data,length);
                         int pos = 0;
                         while (ReadTokenInternal()) {
-                            switch ((JSONGrammar.TokenType)TokenTags[0]) {
+                            switch ((JSONGrammar.TokenType)Tags[0]) {
                                 case JSONGrammar.TokenType.UnescapedChar:
                                     Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
                                     pos += m_tokenLen;
                                     break;
                                 case JSONGrammar.TokenType.EscapedChar:
                                     translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
                                     pos++;
                                     break;
                                 case JSONGrammar.TokenType.EscapedUnicode:
                                     translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
                                     pos++;
                                     break;
                             }
                         }
                         return new String(translated, 0, pos);
                     }
                     internal static char TranslateEscapedChar(char symbol) {
                         return _escMap[symbol];
                     }
                     internal static char TranslateHexUnicode(char[] symbols, int offset) {
                         Debug.Assert(symbols != null);
                         Debug.Assert(symbols.Length - offset >= 4);
                         int value = (_hexMap[symbols[offset]] << 12)
                             | (_hexMap[symbols[offset + 1]] << 8)
                             | (_hexMap[symbols[offset + 2]] << 4)
                             | (_hexMap[symbols[offset + 3]]);
                         return (char)value;
                     }
                 }
             }

Implab/Formats/TextScanner.cs +113 -17

@@ -1,53 +1,149
1	using System;	1	using System;
2	using Implab.Components;	2	using Implab.Components;
3	using Implab.Automaton.RegularExpressions;	3	using Implab.Automaton.RegularExpressions;
4	using System.Diagnostics;	4	using System.Diagnostics;
5	using Implab.Automaton;	5	using Implab.Automaton;
		6	using System.IO;
		7	using System.Text;
6		8
7	namespace Implab.Formats {	9	namespace Implab.Formats {
8	public abstract class TextScanner<~~TTag~~> : Disposable {	10	public abstract class TextScanner : Disposable {
		11	readonly int m_bufferMax;
		12	readonly int m_chunkSize;
9		13
10	int m_maxSymbol;	14	char[] m_buffer;
11	int[] m_symbolMap;
12
13	readonly char[] m_buffer;
14	int m_bufferOffset;	15	int m_bufferOffset;
15	int m_bufferSize;	16	int m_bufferSize;
		17	int m_tokenOffset;
16	int m_tokenLength;	18	int m_tokenLength;
17		19
18	TTag[] m_tags;	20	/// <summary>
		21	/// Initializes a new instance of the <see cref="Implab.Formats.TextScanner`1"/> class.
		22	/// </summary>
		23	/// <param name="bufferMax">Buffer max.</param>
		24	/// <param name="chunkSize">Chunk size.</param>
		25	protected TextScanner(int bufferMax, int chunkSize) {
		26	Debug.Assert(m_chunkSize <= m_bufferMax);
		27
		28	m_bufferMax = bufferMax;
		29	m_chunkSize = chunkSize;
		30	}
19		31
20	protected bool ReadTokenInternal(DFAStateDescriptor<TTag>[] dfa, int state) {	32	/// <summary>
21	Debug.Assert(dfa != null);	33	/// Initializes a new instance of the <see cref="Implab.Formats.TextScanner`1"/> class.
		34	/// </summary>
		35	/// <param name="buffer">Buffer.</param>
		36	protected TextScanner(char[] buffer) {
		37	if (buffer != null) {
		38	m_buffer = buffer;
		39	m_bufferSize = buffer.Length;
		40	}
		41	}
		42
		43	/// <summary>
		44	/// (hungry) Reads the next token.
		45	/// </summary>
		46	/// <returns><c>true</c>, if token internal was read, <c>false</c> if there is no more tokens in the stream.</returns>
		47	/// <param name="dfa">The transition map for the automaton</param>
		48	/// <param name="final">Final states of the automaton.</param>
		49	/// <param name="tags">Tags.</param>
		50	/// <param name="state">The initial state for the automaton.</param>
		51	internal bool ReadToken<TTag>(int[,] dfa, int[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
		52	Safe.ArgumentNotNull();
		53	m_tokenLength = 0;
		54
		55	var maxSymbol = alphabet.Length - 1;
22		56
23	do {	57	do {
24	for (var pos = m_bufferOffset; pos < m_bufferSize; pos++) {	58	// after the next chunk is read the offset in the buffer may change
		59	int pos = m_bufferOffset + m_tokenLength;
		60
		61	while(pos < m_bufferSize) {
25	var ch = m_buffer[pos];	62	var ch = m_buffer[pos];
26	state = dfa[state].transitions[m_symbolMap[ch > m_maxSymbol ? m_maxSymbol : ch]];	63
		64	state = dfa[state,ch > maxSymbol ? DFAConst.UNCLASSIFIED_INPUT : alphabet[ch]];
27	if (state == DFAConst.UNREACHABLE_STATE)	65	if (state == DFAConst.UNREACHABLE_STATE)
28	break;	66	break;
		67
		68	pos++;
29	}	69	}
30	} while (Feed());	70
		71	m_tokenLength = pos - m_bufferOffset;
		72	} while (state != DFAConst.UNREACHABLE_STATE && Feed());
		73
		74	m_tokenOffset = m_bufferOffset;
		75	m_bufferOffset += m_tokenLength;
31		76
32	if (dfa[state].~~final~~) {	77	if (final[state]) {
		78	tag = tags[state];
		79	return true;
		80	} else {
		81	if (m_bufferOffset == m_bufferSize) {
		82	if (m_tokenLength == 0) //EOF
		83	return false;
		84
		85	throw new ParserException();
		86	}
		87	throw new ParserException(String.Format("Unexpected symbol '{0}'", m_buffer[m_bufferOffset]));
33		88
34	}	89	}
		90	}
35		91
		92	protected void Feed(char[] buffer, int offset, int length) {
		93	m_buffer = buffer;
		94	m_bufferOffset = offset;
		95	m_bufferSize = offset + length;
36	}	96	}
37		97
38	bool Feed() {	98	protected bool Feed() {
		99	if (m_chunkSize <= 0)
		100	return false;
		101
		102	if (m_buffer != null) {
		103	var free = m_buffer.Length - m_bufferSize;
		104
		105	if (free < m_chunkSize) {
		106	free += m_chunkSize;
		107	var used = m_bufferSize - m_bufferOffset;
		108	var size = used + free;
		109
		110	if (size > m_bufferMax)
		111	throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached"), m_bufferMax/1024);
		112
		113	var temp = new char[size];
39		114
		115	var read = Read(temp, used, m_chunkSize);
		116	if (read == 0)
		117	return false;
		118
		119	Array.Copy(m_buffer, m_bufferOffset, temp, 0, used);
		120
		121	m_bufferOffset = 0;
		122	m_bufferSize = used + read;
		123	m_buffer = temp;
		124	}
		125	} else {
		126	Debug.Assert(m_bufferOffset == 0);
		127	m_buffer = new char[m_chunkSize];
		128	m_bufferSize = Read(m_buffer, 0, m_chunkSize);
		129	return (m_bufferSize != 0);
		130	}
40	}	131	}
41		132
42	protected abstract int Read(char[] buffer, int offset, int size);	133	protected abstract int Read(char[] buffer, int offset, int size);
43		134
44	protected TTag[] Tags {	135	public string GetTokenValue() {
45	get {	136	return new String(m_buffer, m_tokenOffset, m_tokenLength);
46	return m_tags;
47	}
48	}	137	}
49		138
		139	public void CopyTokenTo(char[] buffer, int offset) {
		140	m_buffer.CopyTo(buffer, offset);
		141	}
		142
		143	public void CopyTokenTo(StringBuilder sb) {
		144	sb.Append(m_buffer, m_tokenOffset, m_tokenLength);
		145	}
50		146
51	}	147	}
52	}	148	}
53		149

Implab/Implab.csproj +3 -4

             <?xml version="1.0" encoding="utf-8"?>
             <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
               <PropertyGroup>
                 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
                 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
                 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
                 <OutputType>Library</OutputType>
                 <RootNamespace>Implab</RootNamespace>
                 <AssemblyName>Implab</AssemblyName>
                 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
                 <ReleaseVersion>0.2</ReleaseVersion>
                 <ProductVersion>8.0.30703</ProductVersion>
                 <SchemaVersion>2.0</SchemaVersion>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>TRACE;DEBUG;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
                 <RunCodeAnalysis>true</RunCodeAnalysis>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
                 <DebugType>full</DebugType>
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug 4.5|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>TRACE;DEBUG;NET_4_5</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <RunCodeAnalysis>true</RunCodeAnalysis>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release 4.5|AnyCPU' ">
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
                 <DefineConstants>NET_4_5</DefineConstants>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'DebugMono|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>TRACE;DEBUG;NET_4_5;MONO</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <RunCodeAnalysis>true</RunCodeAnalysis>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseMono|AnyCPU' ">
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <DefineConstants>NET_4_5;MONO;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <ItemGroup>
                 <Reference Include="System" />
                 <Reference Include="System.Xml" />
                 <Reference Include="mscorlib" />
               </ItemGroup>
               <ItemGroup>
                 <Compile Include="CustomEqualityComparer.cs" />
                 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
                 <Compile Include="Diagnostics\EventText.cs" />
                 <Compile Include="Diagnostics\LogChannel.cs" />
                 <Compile Include="Diagnostics\LogicalOperation.cs" />
                 <Compile Include="Diagnostics\TextFileListener.cs" />
                 <Compile Include="Diagnostics\TraceLog.cs" />
                 <Compile Include="Diagnostics\TraceEvent.cs" />
                 <Compile Include="Diagnostics\TraceEventType.cs" />
                 <Compile Include="ICancellable.cs" />
                 <Compile Include="IProgressHandler.cs" />
                 <Compile Include="IProgressNotifier.cs" />
                 <Compile Include="IPromiseT.cs" />
                 <Compile Include="IPromise.cs" />
                 <Compile Include="IServiceLocator.cs" />
                 <Compile Include="ITaskController.cs" />
                 <Compile Include="Parallels\DispatchPool.cs" />
                 <Compile Include="Parallels\ArrayTraits.cs" />
                 <Compile Include="Parallels\MTQueue.cs" />
                 <Compile Include="Parallels\WorkerPool.cs" />
                 <Compile Include="ProgressInitEventArgs.cs" />
                 <Compile Include="Properties\AssemblyInfo.cs" />
                 <Compile Include="Parallels\AsyncPool.cs" />
                 <Compile Include="Safe.cs" />
                 <Compile Include="ValueEventArgs.cs" />
                 <Compile Include="PromiseExtensions.cs" />
                 <Compile Include="SyncContextPromise.cs" />
                 <Compile Include="Diagnostics\OperationContext.cs" />
                 <Compile Include="Diagnostics\TraceContext.cs" />
                 <Compile Include="Diagnostics\LogEventArgs.cs" />
                 <Compile Include="Diagnostics\LogEventArgsT.cs" />
                 <Compile Include="Diagnostics\Extensions.cs" />
                 <Compile Include="PromiseEventType.cs" />
                 <Compile Include="Parallels\AsyncQueue.cs" />
                 <Compile Include="PromiseT.cs" />
                 <Compile Include="IDeferred.cs" />
                 <Compile Include="IDeferredT.cs" />
                 <Compile Include="Promise.cs" />
                 <Compile Include="PromiseTransientException.cs" />
                 <Compile Include="Parallels\Signal.cs" />
                 <Compile Include="Parallels\SharedLock.cs" />
                 <Compile Include="Diagnostics\ILogWriter.cs" />
                 <Compile Include="Diagnostics\ListenerBase.cs" />
                 <Compile Include="Parallels\BlockingQueue.cs" />
                 <Compile Include="AbstractEvent.cs" />
                 <Compile Include="AbstractPromise.cs" />
                 <Compile Include="AbstractPromiseT.cs" />
                 <Compile Include="FuncTask.cs" />
                 <Compile Include="FuncTaskBase.cs" />
                 <Compile Include="FuncTaskT.cs" />
                 <Compile Include="ActionChainTaskBase.cs" />
                 <Compile Include="ActionChainTask.cs" />
                 <Compile Include="ActionChainTaskT.cs" />
                 <Compile Include="FuncChainTaskBase.cs" />
                 <Compile Include="FuncChainTask.cs" />
                 <Compile Include="FuncChainTaskT.cs" />
                 <Compile Include="ActionTaskBase.cs" />
                 <Compile Include="ActionTask.cs" />
                 <Compile Include="ActionTaskT.cs" />
                 <Compile Include="ICancellationToken.cs" />
                 <Compile Include="SuccessPromise.cs" />
                 <Compile Include="SuccessPromiseT.cs" />
                 <Compile Include="PromiseAwaiterT.cs" />
                 <Compile Include="PromiseAwaiter.cs" />
                 <Compile Include="Components\ComponentContainer.cs" />
                 <Compile Include="Components\Disposable.cs" />
                 <Compile Include="Components\DisposablePool.cs" />
                 <Compile Include="Components\ObjectPool.cs" />
                 <Compile Include="Components\ServiceLocator.cs" />
                 <Compile Include="Components\IInitializable.cs" />
                 <Compile Include="TaskController.cs" />
                 <Compile Include="Components\App.cs" />
                 <Compile Include="Components\IRunnable.cs" />
                 <Compile Include="Components\ExecutionState.cs" />
                 <Compile Include="Components\RunnableComponent.cs" />
                 <Compile Include="Components\IFactory.cs" />
-                <Compile Include="Automaton\DFAStateDescriptor.cs" />
                 <Compile Include="Automaton\EnumAlphabet.cs" />
                 <Compile Include="Automaton\IAlphabet.cs" />
                 <Compile Include="Automaton\ParserException.cs" />
-                <Compile Include="Automaton\Scanner.cs" />
                 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
                 <Compile Include="Automaton\IAlphabetBuilder.cs" />
                 <Compile Include="Automaton\RegularExpressions\AltToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\BinaryToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\CatToken.cs" />
                 <Compile Include="Automaton\DFAConst.cs" />
                 <Compile Include="Automaton\RegularExpressions\Grammar.cs" />
                 <Compile Include="Automaton\RegularExpressions\StarToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\SymbolToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\EmptyToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
                 <Compile Include="Automaton\RegularExpressions\Token.cs" />
                 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
                 <Compile Include="Automaton\AutomatonTransition.cs" />
                 <Compile Include="Formats\JSON\JSONElementContext.cs" />
                 <Compile Include="Formats\JSON\JSONElementType.cs" />
                 <Compile Include="Formats\JSON\JSONGrammar.cs" />
                 <Compile Include="Formats\JSON\JSONParser.cs" />
                 <Compile Include="Formats\JSON\JSONScanner.cs" />
                 <Compile Include="Formats\JSON\JsonTokenType.cs" />
                 <Compile Include="Formats\JSON\JSONWriter.cs" />
                 <Compile Include="Formats\JSON\JSONXmlReader.cs" />
                 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
                 <Compile Include="Formats\JSON\StringTranslator.cs" />
                 <Compile Include="Automaton\MapAlphabet.cs" />
                 <Compile Include="Automaton\DummyAlphabet.cs" />
                 <Compile Include="Formats\CharAlphabet.cs" />
                 <Compile Include="Formats\ByteAlphabet.cs" />
                 <Compile Include="Automaton\IDFATable.cs" />
                 <Compile Include="Automaton\IDFATableBuilder.cs" />
                 <Compile Include="Automaton\DFATable.cs" />
                 <Compile Include="Automaton\RegularExpressions\RegularDFA.cs" />
                 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
                 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
-                <Compile Include="Automaton\RegularExpressions\DFAStateDescriptorT.cs" />
-                <Compile Include="Formats\BufferScanner.cs" />
                 <Compile Include="Formats\TextScanner.cs" />
+                <Compile Include="Formats\StringScanner.cs" />
+                <Compile Include="Formats\ReaderScanner.cs" />
+                <Compile Include="Formats\ScannerContext.cs" />
               </ItemGroup>
               <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
               <ItemGroup />
               <ProjectExtensions>
                 <MonoDevelop>
                   <Properties>
                     <Policies>
                       <CSharpFormattingPolicy IndentSwitchBody="True" NamespaceBraceStyle="EndOfLine" ClassBraceStyle="EndOfLine" InterfaceBraceStyle="EndOfLine" StructBraceStyle="EndOfLine" EnumBraceStyle="EndOfLine" MethodBraceStyle="EndOfLine" ConstructorBraceStyle="EndOfLine" DestructorBraceStyle="EndOfLine" BeforeMethodDeclarationParentheses="False" BeforeMethodCallParentheses="False" BeforeConstructorDeclarationParentheses="False" NewLineBeforeConstructorInitializerColon="NewLine" NewLineAfterConstructorInitializerColon="SameLine" BeforeIndexerDeclarationBracket="False" BeforeDelegateDeclarationParentheses="False" NewParentheses="False" SpacesBeforeBrackets="False" inheritsSet="Mono" inheritsScope="text/x-csharp" scope="text/x-csharp" />
                       <TextStylePolicy FileWidth="120" EolMarker="Unix" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/x-csharp" />
                       <DotNetNamingPolicy DirectoryNamespaceAssociation="PrefixedHierarchical" ResourceNamePolicy="MSBuild" />
                       <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="application/xml" />
                       <XmlFormattingPolicy inheritsSet="Mono" inheritsScope="application/xml" scope="application/xml" />
                       <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/plain" />
                       <NameConventionPolicy>
                         <Rules>
                           <NamingRule Name="Namespaces" AffectedEntity="Namespace" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Types" AffectedEntity="Class, Struct, Enum, Delegate" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Interfaces" AffectedEntity="Interface" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
                             <RequiredPrefixes>
                               <String>I</String>
                             </RequiredPrefixes>
                           </NamingRule>
                           <NamingRule Name="Attributes" AffectedEntity="CustomAttributes" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
                             <RequiredSuffixes>
                               <String>Attribute</String>
                             </RequiredSuffixes>
                           </NamingRule>
                           <NamingRule Name="Event Arguments" AffectedEntity="CustomEventArgs" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
                             <RequiredSuffixes>
                               <String>EventArgs</String>
                             </RequiredSuffixes>
                           </NamingRule>
                           <NamingRule Name="Exceptions" AffectedEntity="CustomExceptions" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
                             <RequiredSuffixes>
                               <String>Exception</String>
                             </RequiredSuffixes>
                           </NamingRule>
                           <NamingRule Name="Methods" AffectedEntity="Methods" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Static Readonly Fields" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Protected, Public" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True" />
                           <NamingRule Name="Fields (Non Private)" AffectedEntity="Field" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="ReadOnly Fields (Non Private)" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False" />
                           <NamingRule Name="Fields (Private)" AffectedEntity="Field, ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
                             <RequiredPrefixes>
                               <String>m_</String>
                             </RequiredPrefixes>
                           </NamingRule>
                           <NamingRule Name="Static Fields (Private)" AffectedEntity="Field" VisibilityMask="Private" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True">
                             <RequiredPrefixes>
                               <String>_</String>
                             </RequiredPrefixes>
                           </NamingRule>
                           <NamingRule Name="ReadOnly Fields (Private)" AffectedEntity="ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
                             <RequiredPrefixes>
                               <String>m_</String>
                             </RequiredPrefixes>
                           </NamingRule>
                           <NamingRule Name="Constant Fields" AffectedEntity="ConstantField" VisibilityMask="VisibilityMask" NamingStyle="AllUpper" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Properties" AffectedEntity="Property" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Events" AffectedEntity="Event" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Enum Members" AffectedEntity="EnumMember" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Parameters" AffectedEntity="Parameter, LocalVariable" VisibilityMask="VisibilityMask" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
                           <NamingRule Name="Type Parameters" AffectedEntity="TypeParameter" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
                             <RequiredPrefixes>
                               <String>T</String>
                             </RequiredPrefixes>
                           </NamingRule>
                         </Rules>
                       </NameConventionPolicy>
                     </Policies>
                   </Properties>
                 </MonoDevelop>
               </ProjectExtensions>
               <ItemGroup>
                 <Folder Include="Components\" />
                 <Folder Include="Automaton\RegularExpressions\" />
                 <Folder Include="Formats\" />
                 <Folder Include="Formats\JSON\" />
               </ItemGroup>
             </Project>

Implab/Automaton/DFAStateDescriptor.cs removed 0 -26

NO CONTENT: file was removed

Implab/Automaton/RegularExpressions/DFAStateDescriptorT.cs removed 0 -23

NO CONTENT: file was removed

Implab/Automaton/Scanner.cs removed 0 -255

NO CONTENT: file was removed

Implab/Formats/BufferScanner.cs removed 0 -62

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages