implab/ImplabNet Commit - r181:b2b6a6640aa3

minor fixes and debug

cin -

r181:b2b6a6640aa3 ref20160224

parent child

Context file:

r181:b2b6a6640aa3

Collapse all files

Implab/Automaton/DFATable.cs +10 -7

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
+            using System.Diagnostics;
             namespace Implab.Automaton {
                 public class DFATable : IDFATableBuilder {
                     int m_stateCount;
                     int m_symbolCount;
                     int m_initialState;
                     readonly HashSet<int> m_finalStates = new HashSet<int>();
                     readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
                     #region IDFADefinition implementation
                     public bool IsFinalState(int s) {
                         Safe.ArgumentInRange(s, 0, m_stateCount, "s");
                         return m_finalStates.Contains(s);
                     }
                     public IEnumerable<int> FinalStates {
                         get {
                             return m_finalStates;
                         }
                     }
                     public int StateCount {
                         get { return m_stateCount; }
                     }
                     public int AlphabetSize {
                         get { return m_symbolCount; }
                     }
                     public int InitialState {
                         get { return m_initialState; }
                     }
                     #endregion
                     public void SetInitialState(int s) {
                         Safe.ArgumentAssert(s >= 0, "s");
+                        m_stateCount = Math.Max(m_stateCount, s + 1);
                         m_initialState = s;
                     }
                     public void MarkFinalState(int state) {
+                        m_stateCount = Math.Max(m_stateCount, state + 1);
                         m_finalStates.Add(state);
                     }
                     public void Add(AutomatonTransition item) {
                         Safe.ArgumentAssert(item.s1 >= 0, "item");
                         Safe.ArgumentAssert(item.s2 >= 0, "item");
                         Safe.ArgumentAssert(item.edge >= 0, "item");
                         m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
-                        m_symbolCount = Math.Max(m_symbolCount, item.edge);
+                        m_symbolCount = Math.Max(m_symbolCount, item.edge + 1);
                         m_transitions.Add(item);
                     }
                     public void Clear() {
                         m_stateCount = 0;
                         m_symbolCount = 0;
                         m_finalStates.Clear();
                         m_transitions.Clear();
                     }
                     public bool Contains(AutomatonTransition item) {
                         return m_transitions.Contains(item);
                     }
                     public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
                         m_transitions.CopyTo(array, arrayIndex);
                     }
                     public bool Remove(AutomatonTransition item) {
                         return m_transitions.Remove(item);
                     }
                     public int Count {
                         get {
                             return m_transitions.Count;
                         }
                     }
                     public bool IsReadOnly {
                         get {
                             return false;
                         }
                     }
                     public IEnumerator<AutomatonTransition> GetEnumerator() {
                         return m_transitions.GetEnumerator();
                     }
                     System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
                         return GetEnumerator();
                     }
                     public int[,] CreateTransitionTable() {
                         var table = new int[StateCount,AlphabetSize];
                         for (int i = 0; i < StateCount; i++)
-                            for (int j = 0; i < AlphabetSize; j++)
+                            for (int j = 0; j < AlphabetSize; j++)
                                 table[i, j] = AutomatonConst.UNREACHABLE_STATE;
                         foreach (var t in this)
                             table[t.s1,t.edge] = t.s2;
                         return table;
                     }
                     public bool[] CreateFinalStateTable() {
                         var table = new bool[StateCount];
                         foreach (var s in FinalStates)
                             table[s] = true;
                         return table;
                     }
                     /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
                     /// <remarks>
                     /// В процессе построения минимального автомата требуется разделить множество состояний,
                     /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
                     /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
                     /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
                     /// </remarks>
                     /// <returns>The final states.</returns>
                     protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
                         return new HashSet<int>[] { m_finalStates };
                     }
                     protected void Optimize(
                         IDFATableBuilder optimalDFA,
                         IDictionary<int,int> alphabetMap,
                         IDictionary<int,int> stateMap
                     ) {
                         Safe.ArgumentNotNull(optimalDFA, "dfa");
                         Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
                         Safe.ArgumentNotNull(stateMap, "stateMap");
                         var setComparer = new CustomEqualityComparer<HashSet<int>>(
                             (x, y) => x.SetEquals(y),
                             s => s.Sum(x => x.GetHashCode())
                         );
                         var optimalStates = new HashSet<HashSet<int>>(setComparer);
                         var queue = new HashSet<HashSet<int>>(setComparer);
                         // получаем конечные состояния, сгруппированные по маркерам
                         optimalStates.UnionWith(
                             GroupFinalStates()
                         );
                         var state = new HashSet<int>(
                             Enumerable
                             .Range(0, m_stateCount - 1)
                             .Where(i => !m_finalStates.Contains(i))
                         );
                         optimalStates.Add(state);
                         queue.Add(state);
                         var rmap = m_transitions
                             .GroupBy(t => t.s2)
                             .ToDictionary(
                                 g => g.Key, // s2
-                                g => g.GroupBy(t => t.edge, t => t.s1).ToDictionary(p => p.Key)
+                                g => g.ToLookup(t => t.edge, t => t.s1)//.ToDictionary(p => p.Key)
                             );
                         while (queue.Count > 0) {
                             var stateA = queue.First();
                             queue.Remove(stateA);
                             for (int c = 0; c < m_symbolCount; c++) {
                                 var stateX = new HashSet<int>();
-                                foreach(var a in stateA)
+                                foreach(var a in stateA.Where(rmap.ContainsKey))
                                     stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
                                 foreach (var stateY in optimalStates.ToArray()) {
                                     if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
                                         var stateR1 = new HashSet<int>(stateY);
                                         var stateR2 = new HashSet<int>(stateY);
                                         stateR1.IntersectWith(stateX);
                                         stateR2.ExceptWith(stateX);
                                         optimalStates.Remove(stateY);
                                         optimalStates.Add(stateR1);
                                         optimalStates.Add(stateR2);
                                         if (queue.Contains(stateY)) {
                                             queue.Remove(stateY);
                                             queue.Add(stateR1);
                                             queue.Add(stateR2);
                                         } else {
                                             queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
                                         }
                                     }
                                 }
                             }
                         }
                         // карта получения оптимального состояния по соотвествующему ему простому состоянию
                         var nextState = 0;
                         foreach (var item in optimalStates) {
                             var id = nextState++;
                             foreach (var s in item)
                                 stateMap[s] = id;
                         }
                         // получаем минимальный алфавит
                         // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
                         // для этого используем алгоритм кластеризации, сначала
                         // считаем, что все символы не различимы
                         var minClasses = new HashSet<HashSet<int>>(setComparer);
                         var alphaQueue = new Queue<HashSet<int>>();
                         alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
                         // для всех состояний, будем проверять каждый класс на различимость,
                         // т.е. символы различимы, если они приводят к разным состояниям
                         for (int s = 0 ; s < optimalStates.Count; s++) {
                             var newQueue = new Queue<HashSet<int>>();
                             foreach (var A in alphaQueue) {
                                 // классы из одного символа делить бесполезно, переводим их сразу в
                                 // результирующий алфавит
                                 if (A.Count == 1) {
                                     minClasses.Add(A);
                                     continue;
                                 }
                                 // различаем классы символов, которые переводят в различные оптимальные состояния
                                 // optimalState -> alphaClass
                                 var classes = new Dictionary<int, HashSet<int>>();
                                 foreach (var term in A) {
                                     // ищем все переходы класса по символу term
                                     var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray();
+                                    Debug.Assert(res.Length <= 1);
                                     var s2 = res.Length > 0 ? res[0] : -1;
                                     HashSet<int> a2;
                                     if (!classes.TryGetValue(s2, out a2)) {
                                         a2 = new HashSet<int>();
                                         newQueue.Enqueue(a2);
                                         classes[s2] = a2;
                                     }
                                     a2.Add(term);
                                 }
                             }
                             if (newQueue.Count == 0)
                                 break;
                             alphaQueue = newQueue;
                         }
                         // после окончания работы алгоритма в очереди останутся минимальные различимые классы
                         // входных символов
                         foreach (var A in alphaQueue)
                             minClasses.Add(A);
                         // построение отображения алфавитов входных символов.
                         // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
                         // специальное значение, тогда сохраним минимальный класс,
                         // содержащий этот символ на томже месте.
                         var nextCls = 0;
                         foreach (var item in minClasses) {
                             if (nextCls == AutomatonConst.UNCLASSIFIED_INPUT)
                                 nextCls++;
                             // сохраняем DFAConst.UNCLASSIFIED_INPUT
-                            var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls;
+                            var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++;
                             foreach (var a in item)
                                 alphabetMap[a] = cls;
-                            nextCls++;
                         }
                         // построение автомата
                         optimalDFA.SetInitialState(stateMap[m_initialState]);
                         foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
                             optimalDFA.MarkFinalState(sf);
                         foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
                             optimalDFA.Add(t);
                     }
                     protected void PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
                         Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
                         Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
                         foreach(var t in m_transitions)
                             Console.WriteLine(
                                 "[{0}] -{{{1}}}-> [{2}]{3}",
                                 String.Join(",", stateAlphabet.GetSymbols(t.s1)),
                                 String.Join("", inputAlphabet.GetSymbols(t.edge)),
                                 String.Join(",", stateAlphabet.GetSymbols(t.s2)),
                                 m_finalStates.Contains(t.s2) ? "$" : ""
                             );
                     }
                 }
             }

Implab/Automaton/MapAlphabet.cs +1 -1

             using System;
             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton {
                 public class MapAlphabet<T> : IAlphabetBuilder<T> {
                     readonly Dictionary<T,int> m_map;
                     int m_nextCls;
                     readonly bool m_supportUnclassified;
                     public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
                         m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
                         m_supportUnclassified = supportUnclassified;
                         m_nextCls = supportUnclassified ? 1 : 0;
                     }
                     #region IAlphabetBuilder implementation
                     public int DefineSymbol(T symbol) {
                         int cls;
                         return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
                     }
                     public int DefineSymbol(T symbol, int cls) {
                         Safe.ArgumentAssert(cls >= 0, "cls");
                         m_nextCls = Math.Max(cls + 1, m_nextCls);
                         m_map.Add(symbol, cls);
                         return cls;
                     }
                     public int DefineClass(IEnumerable<T> symbols) {
                         return DefineClass(symbols, m_nextCls);
                     }
                     public int DefineClass(IEnumerable<T> symbols, int cls) {
                         Safe.ArgumentAssert(cls >= 0, "cls");
                         Safe.ArgumentNotNull(symbols, "symbols");
                         m_nextCls = Math.Max(cls + 1, m_nextCls);
                         foreach (var symbol in symbols)
                             m_map[symbol] = cls;
                         return cls;
                     }
                     #endregion
                     #region IAlphabet implementation
                     public int Translate(T symbol) {
                         int cls;
                         if (m_map.TryGetValue(symbol, out cls))
                             return cls;
                         if (!m_supportUnclassified)
                             throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
                         return AutomatonConst.UNCLASSIFIED_INPUT;
                     }
                     public int Count {
                         get {
                             return m_nextCls;
                         }
                     }
                     public bool Contains(T symbol) {
                         return m_supportUnclassified || m_map.ContainsKey(symbol);
                     }
                     public IEnumerable<T> GetSymbols(int cls) {
-                        Safe.ArgumentAssert(cls > 0, "cls");
+                        Safe.ArgumentAssert(!m_supportUnclassified || cls > 0, "cls");
                         return m_map.Where(p => p.Value == cls).Select(p => p.Key);
                     }
                     #endregion
                     public IEnumerable<KeyValuePair<T,int>> Mappings {
                         get {
                             return m_map;
                         }
                     }
                 }
             }

Implab/Automaton/RegularExpressions/RegularDFA.cs +2 -1

             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton.RegularExpressions {
                 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
                     readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
                     readonly IAlphabet<TInput> m_alphabet;
                     public RegularDFA(IAlphabet<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "aplhabet");
                         m_alphabet = alphabet;
                     }
                     public IAlphabet<TInput> InputAlphabet {
                         get {
                             return m_alphabet;
                         }
                     }
                     public void MarkFinalState(int s, TTag[] tags) {
                         MarkFinalState(s);
                         SetStateTag(s, tags);
                     }
                     public void SetStateTag(int s, TTag[] tags) {
                         Safe.ArgumentNotNull(tags, "tags");
                         m_tags[s] = tags;
                     }
                     public TTag[] GetStateTag(int s) {
                         TTag[] tags;
                         return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
                     }
                     public TTag[][] CreateTagTable() {
                         var table = new TTag[StateCount][];
                         foreach (var pair in m_tags)
                             table[pair.Key] = pair.Value;
                         return table;
                     }
                     /// <summary>
                     /// Optimize the specified alphabet.
                     /// </summary>
                     /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
                     public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "alphabet");
                         var dfa = new RegularDFA<TInput, TTag>(alphabet);
                         var alphaMap = new Dictionary<int,int>();
                         var stateMap = new Dictionary<int,int>();
                         Optimize(dfa, alphaMap, stateMap);
                         // mark tags in the new DFA
                         foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
                             dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
                         // make the alphabet for the new DFA
-                        foreach (var pair in alphaMap)
+                        // skip all unclassified symbols
+                        foreach (var pair in alphaMap.Where(x => x.Value != 0))
                             alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
                         return dfa;
                     }
                     protected override IEnumerable<HashSet<int>> GroupFinalStates() {
                         var arrayComparer = new CustomEqualityComparer<TTag[]>(
                             (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
                             x => x.Sum(it => x.GetHashCode())
                         );
                         return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
                     }
                 }
             }

Implab/Formats/JSON/JSONGrammar.cs +1 -1

             using System.Linq;
             using Implab.Automaton.RegularExpressions;
             using System;
             using Implab.Automaton;
             using Implab.Components;
             namespace Implab.Formats.JSON {
                 class JSONGrammar : Grammar<char> {
                     public enum TokenType {
                         None,
                         BeginObject,
                         EndObject,
                         BeginArray,
                         EndArray,
                         String,
                         Number,
                         Literal,
                         NameSeparator,
                         ValueSeparator,
                         StringBound,
                         EscapedChar,
                         UnescapedChar,
                         EscapedUnicode
                     }
                     static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
                     public static JSONGrammar Instance {
                         get { return _instance.Value; }
                     }
                     readonly ScannerContext<TokenType> m_jsonExpression;
                     readonly ScannerContext<TokenType> m_stringExpression;
                     readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
                     public JSONGrammar() {
                         DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
                         var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
                         var digit9 = SymbolRangeToken('1', '9');
                         var zero = SymbolToken('0');
                         var digit = zero.Or(digit9);
                         var dot = SymbolToken('.');
                         var minus = SymbolToken('-');
                         var sign = SymbolSetToken('-', '+');
                         var expSign = SymbolSetToken('e', 'E');
                         var letters = SymbolRangeToken('a', 'z');
                         var integer = zero.Or(digit9.Cat(digit.EClosure()));
                         var frac = dot.Cat(digit.Closure());
                         var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
                         var quote = SymbolToken('"');
                         var backSlash = SymbolToken('\\');
                         var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
                         var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
                         var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
                         var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
                         var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
                         var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
                         var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
                         var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
                         var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
                         var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
                         var literal = letters.Closure();
                         var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
                         var jsonExpression =
                             number.Tag(TokenType.Number)
                             .Or(literal.Tag(TokenType.Literal))
                             .Or(quote.Tag(TokenType.StringBound))
                             .Or(beginObject.Tag(TokenType.BeginObject))
                             .Or(endObject.Tag(TokenType.EndObject))
                             .Or(beginArray.Tag(TokenType.BeginArray))
                             .Or(endArray.Tag(TokenType.EndArray))
                             .Or(nameSep.Tag(TokenType.NameSeparator))
                             .Or(valueSep.Tag(TokenType.ValueSeparator));
                         var jsonStringExpression =
                             quote.Tag(TokenType.StringBound)
                             .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
                             .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
                             .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
                         m_jsonExpression = BuildScannerContext<TokenType>(jsonExpression);
                         m_stringExpression = BuildScannerContext<TokenType>(jsonStringExpression);
                     }
                     protected override IAlphabetBuilder<char> AlphabetBuilder {
                         get {
                             return m_defaultAlphabet;
                         }
                     }
                     public ScannerContext<TokenType> JsonExpression {
                         get {
                             return m_jsonExpression;
                         }
                     }
                     public ScannerContext<TokenType> JsonStringExpression {
                         get {
                             return m_stringExpression;
                         }
                     }
                     Token SymbolRangeToken(char start, char stop) {
-                        return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
+                        return SymbolToken(Enumerable.Range(start,stop - start).Select(x => (char)x));
                     }
                     protected override IndexedAlphabetBase<char> CreateAlphabet() {
                         return new CharAlphabet();
                     }
                 }
             }

Implab/Formats/TextScanner.cs +8 -2

             using System;
             using Implab.Components;
             using System.Diagnostics;
             using Implab.Automaton;
             using System.Text;
             namespace Implab.Formats {
                 public abstract class TextScanner : Disposable {
                     readonly int m_bufferMax;
                     readonly int m_chunkSize;
                     char[] m_buffer;
                     int m_bufferOffset;
                     int m_bufferSize;
                     int m_tokenOffset;
                     int m_tokenLength;
                     /// <summary>
                     /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner"/> class.
                     /// </summary>
                     /// <param name="bufferMax">Buffer max.</param>
                     /// <param name="chunkSize">Chunk size.</param>
                     protected TextScanner(int bufferMax, int chunkSize) {
                         Debug.Assert(m_chunkSize <= m_bufferMax);
                         m_bufferMax = bufferMax;
                         m_chunkSize = chunkSize;
                     }
                     /// <summary>
                     /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner"/> class.
                     /// </summary>
                     /// <param name="buffer">Buffer.</param>
                     protected TextScanner(char[] buffer) {
                         if (buffer != null) {
                             m_buffer = buffer;
                             m_bufferSize = buffer.Length;
                         }
                     }
                     /// <summary>
                     /// (hungry) Reads the next token.
                     /// </summary>
                     /// <returns><c>true</c>, if token internal was read, <c>false</c> if there is no more tokens in the stream.</returns>
                     /// <param name="dfa">The transition map for the automaton</param>
                     /// <param name="final">Final states of the automaton.</param>
                     /// <param name="tags">Tags.</param>
                     /// <param name="state">The initial state for the automaton.</param>
                     /// <param name="alphabet"></param>
                     /// <param name = "tag"></param>
                     internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
                         m_tokenLength = 0;
                         tag = null;
                         var maxSymbol = alphabet.Length - 1;
                         do {
                             // after the next chunk is read the offset in the buffer may change
                             int pos = m_bufferOffset + m_tokenLength;
                             while (pos < m_bufferSize) {
                                 var ch = m_buffer[pos];
-                                state = dfa[state, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
+                                try {
-                                if (state == AutomatonConst.UNREACHABLE_STATE)
+                                var next = dfa[state, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
+                                if (next == AutomatonConst.UNREACHABLE_STATE)
                                     break;
+                                state = next;
+                                }catch {
+                                    throw;
+                                }
                                 pos++;
                             }
                             m_tokenLength = pos - m_bufferOffset;
                         } while (state != AutomatonConst.UNREACHABLE_STATE && Feed());
                         m_tokenOffset = m_bufferOffset;
                         m_bufferOffset += m_tokenLength;
                         if (final[state]) {
                             tag = tags[state];
                             return true;
                         }
                         if (m_bufferOffset == m_bufferSize) {
                             if (m_tokenLength == 0) //EOF
                                     return false;
                             throw new ParserException();
                         }
                         throw new ParserException(String.Format("Unexpected symbol '{0}'", m_buffer[m_bufferOffset]));
                     }
                     protected void Feed(char[] buffer, int offset, int length) {
                         m_buffer = buffer;
                         m_bufferOffset = offset;
                         m_bufferSize = offset + length;
                     }
                     protected bool Feed() {
                         if (m_chunkSize <= 0)
                             return false;
                         if (m_buffer != null) {
                             var free = m_buffer.Length - m_bufferSize;
                             if (free < m_chunkSize) {
                                 free += m_chunkSize;
                                 var used = m_bufferSize - m_bufferOffset;
                                 var size = used + free;
                                 if (size > m_bufferMax)
                                     throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax / 1024));
                                 var temp = new char[size];
                                 var read = Read(temp, used, m_chunkSize);
                                 if (read == 0)
                                     return false;
                                 Array.Copy(m_buffer, m_bufferOffset, temp, 0, used);
                                 m_bufferOffset = 0;
                                 m_bufferSize = used + read;
                                 m_buffer = temp;
                             } else {
                                 var read = Read(m_buffer, m_bufferSize, m_chunkSize);
                                 if (read == 0)
                                     return false;
                                 m_bufferSize += m_chunkSize;
                             }
                             return true;
                         } else {
                             Debug.Assert(m_bufferOffset == 0);
                             m_buffer = new char[m_chunkSize];
                             m_bufferSize = Read(m_buffer, 0, m_chunkSize);
                             return (m_bufferSize != 0);
                         }
                     }
                     protected abstract int Read(char[] buffer, int offset, int size);
                     public string GetTokenValue() {
                         return new String(m_buffer, m_tokenOffset, m_tokenLength);
                     }
                     public void CopyTokenTo(char[] buffer, int offset) {
                         m_buffer.CopyTo(buffer, offset);
                     }
                     public void CopyTokenTo(StringBuilder sb) {
                         sb.Append(m_buffer, m_tokenOffset, m_tokenLength);
                     }
                 }
             }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages