implab/ImplabNet Commit - r183:4f82e0f161c3

fixed DFA optimization, JSON is fully functional

cin -

r183:4f82e0f161c3 ref20160224

parent child

Context file:

r183:4f82e0f161c3

Collapse all files

The requested changes are too big and content was truncated. Show full diff

MonoPlay/packages.config created 644 +4 0

@@ -0,0 +1,4
	1	<?xml version="1.0" encoding="utf-8"?>
	2	<packages>
	3	<package id="System.Text.Json" version="2.0.0.11" targetFramework="net45" />
	4	</packages> No newline at end of file

packages/NUnit.2.6.4/NUnit.2.6.4.nupkg created 644 binary 0 0

NO CONTENT: new file 100644, binary diff hidden

packages/NUnit.2.6.4/lib/nunit.framework.dll created 644 binary 0 0

NO CONTENT: new file 100644, binary diff hidden

packages/NUnit.2.6.4/lib/nunit.framework.xml created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

packages/NUnit.2.6.4/license.txt created 644 0 0

	1	NO CONTENT: new file 100644			NO CONTENT: new file 100644
The requested commit or file is too big and content was truncated. Show full diff

packages/System.Text.Json.2.0.0.11/System.Text.Json.2.0.0.11.nupkg created 644 binary 0 0

NO CONTENT: new file 100644, binary diff hidden

packages/System.Text.Json.2.0.0.11/lib/net40/System.Text.Json.dll created 644 binary 0 0

NO CONTENT: new file 100644, binary diff hidden

Implab.Test/Implab.Format.Test/JsonTests.cs +39 -1

             using NUnit.Framework;
             using System;
             using Implab.Formats.JSON;
+            using Implab.Automaton;
             namespace Implab.Format.Test {
                 [TestFixture]
                 public class JsonTests {
                     [Test]
                     public void TestScannerValidTokens() {
-                        var scanner = new JSONScanner(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:");
+                        using (var scanner = new JSONScanner(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
                             Tuple<JsonTokenType,object>[] expexted = new [] {
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 9123d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -123d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0.1d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.2d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.1e3d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 1.3E-3d),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n  text"),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal"),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, " ["),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, "]"),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, "{"),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, "}"),
                                 new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, ":")
                             };
                             object value;
                             JsonTokenType tokenType;
                             for (var i = 0; i < expexted.Length; i++) {
                                 Assert.IsTrue(scanner.ReadToken(out value, out tokenType));
                                 Assert.AreEqual(expexted[i].Item1, tokenType);
                                 Assert.AreEqual(expexted[i].Item2, value);
                             }
                             Assert.IsFalse(scanner.ReadToken(out value, out tokenType));
                         }
                     }
+                    [Test]
+                    public void TestScannerBadTokens() {
+                        var bad = new [] {
+                            " 1",
+                            " literal",
+                            " \"",
+                            "\"unclosed string",
+                            "1.bad",
+                            "001", // should be read as three numbers
+                            "--10",
+                            "+10",
+                            "1.0.0",
+                            "1e1.0",
+                            "l1teral0",
+                            ".123",
+                            "-.123"
+                        };
+                        foreach (var json in bad)
+                            using (var scanner = new JSONScanner(json)) {
+                                try {
+                                    object value;
+                                    JsonTokenType token;
+                                    scanner.ReadToken(out value, out token);
+                                    if (!Object.Equals(value,json)) {
+                                        Console.WriteLine("'{0}' is read as {1}", json, value is String ? String.Format("'{0}'", value) : value );
+                                        continue;
+                                    }
+                                    Assert.Fail("Token '{0}' shouldn't pass", json);
+                                } catch (ParserException e) {
+                                    Console.WriteLine(e.Message);
+                                }
                             }
+                    }
+                }
+            }

Implab/Automaton/DFATable.cs +16 -11

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             using System.Diagnostics;
             using System.IO;
             using System.CodeDom.Compiler;
             using System.CodeDom;
             namespace Implab.Automaton {
                 public class DFATable : IDFATableBuilder {
                     int m_stateCount;
                     int m_symbolCount;
                     int m_initialState;
                     readonly HashSet<int> m_finalStates = new HashSet<int>();
                     readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
                     #region IDFADefinition implementation
                     public bool IsFinalState(int s) {
                         Safe.ArgumentInRange(s, 0, m_stateCount, "s");
                         return m_finalStates.Contains(s);
                     }
                     public IEnumerable<int> FinalStates {
                         get {
                             return m_finalStates;
                         }
                     }
                     public int StateCount {
                         get { return m_stateCount; }
                     }
                     public int AlphabetSize {
                         get { return m_symbolCount; }
                     }
                     public int InitialState {
                         get { return m_initialState; }
                     }
                     #endregion
                     public void SetInitialState(int s) {
                         Safe.ArgumentAssert(s >= 0, "s");
                         m_stateCount = Math.Max(m_stateCount, s + 1);
                         m_initialState = s;
                     }
                     public void MarkFinalState(int state) {
                         m_stateCount = Math.Max(m_stateCount, state + 1);
                         m_finalStates.Add(state);
                     }
                     public void Add(AutomatonTransition item) {
                         Safe.ArgumentAssert(item.s1 >= 0, "item");
                         Safe.ArgumentAssert(item.s2 >= 0, "item");
                         Safe.ArgumentAssert(item.edge >= 0, "item");
                         m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
                         m_symbolCount = Math.Max(m_symbolCount, item.edge + 1);
                         m_transitions.Add(item);
                     }
                     public void Clear() {
                         m_stateCount = 0;
                         m_symbolCount = 0;
                         m_finalStates.Clear();
                         m_transitions.Clear();
                     }
                     public bool Contains(AutomatonTransition item) {
                         return m_transitions.Contains(item);
                     }
                     public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
                         m_transitions.CopyTo(array, arrayIndex);
                     }
                     public bool Remove(AutomatonTransition item) {
                         return m_transitions.Remove(item);
                     }
                     public int Count {
                         get {
                             return m_transitions.Count;
                         }
                     }
                     public bool IsReadOnly {
                         get {
                             return false;
                         }
                     }
                     public IEnumerator<AutomatonTransition> GetEnumerator() {
                         return m_transitions.GetEnumerator();
                     }
                     System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
                         return GetEnumerator();
                     }
                     public void AddSymbol(int symbol) {
                         Safe.ArgumentAssert(symbol >= 0, "symbol");
                         m_symbolCount = Math.Max(symbol + 1, m_symbolCount);
                     }
                     public int[,] CreateTransitionTable() {
                         var table = new int[StateCount,AlphabetSize];
                         for (int i = 0; i < StateCount; i++)
                             for (int j = 0; j < AlphabetSize; j++)
                                 table[i, j] = AutomatonConst.UNREACHABLE_STATE;
                         foreach (var t in this)
                             table[t.s1,t.edge] = t.s2;
                         return table;
                     }
                     public bool[] CreateFinalStateTable() {
                         var table = new bool[StateCount];
                         foreach (var s in FinalStates)
                             table[s] = true;
                         return table;
                     }
                     /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
                     /// <remarks>
                     /// В процессе построения минимального автомата требуется разделить множество состояний,
                     /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
                     /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
                     /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
                     /// </remarks>
                     /// <returns>The final states.</returns>
-                    protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
+                    protected virtual IEnumerable<HashSet<int>> SplitFinalStates(IEnumerable<int> states) {
-                        return new HashSet<int>[] { m_finalStates };
+                        return new [] { new HashSet<int>(states) };
                     }
                     protected void Optimize(
                         IDFATableBuilder optimalDFA,
                         IDictionary<int,int> alphabetMap,
                         IDictionary<int,int> stateMap
                     ) {
                         Safe.ArgumentNotNull(optimalDFA, "dfa");
                         Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
                         Safe.ArgumentNotNull(stateMap, "stateMap");
                         var setComparer = new CustomEqualityComparer<HashSet<int>>(
                             (x, y) => x.SetEquals(y),
                             s => s.Sum(x => x.GetHashCode())
                         );
                         var optimalStates = new HashSet<HashSet<int>>(setComparer);
                         var queue = new HashSet<HashSet<int>>(setComparer);
-                        // получаем конечные состояния, сгруппированные по маркерам
+                        optimalStates.Add(new HashSet<int>(FinalStates));
-                        optimalStates.UnionWith(
-                            GroupFinalStates()
-                        );
                         var state = new HashSet<int>(
                             Enumerable
                             .Range(0, m_stateCount)
                             .Where(i => !m_finalStates.Contains(i))
                         );
                         optimalStates.Add(state);
                         queue.Add(state);
                         var rmap = m_transitions
                             .GroupBy(t => t.s2)
                             .ToDictionary(
                                 g => g.Key, // s2
                                 g => g.ToLookup(t => t.edge, t => t.s1)//.ToDictionary(p => p.Key)
                             );
                         while (queue.Count > 0) {
                             var stateA = queue.First();
                             queue.Remove(stateA);
                             for (int c = 0; c < m_symbolCount; c++) {
                                 var stateX = new HashSet<int>();
-                                //foreach(var a in stateA.Where(rmap.ContainsKey))
+                                foreach(var a in stateA.Where(rmap.ContainsKey))
-                                //    stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
+                                    stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
-                                stateX.UnionWith(m_transitions.Where(t => stateA.Contains(t.s2) && t.edge == c).Select(t => t.s1));
                                 var tmp = optimalStates.ToArray();
                                 foreach (var stateY in tmp) {
-                                    if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
                                     var stateR1 = new HashSet<int>(stateY);
                                     var stateR2 = new HashSet<int>(stateY);
                                     stateR1.IntersectWith(stateX);
                                     stateR2.ExceptWith(stateX);
+                                    if (stateR1.Count > 0 && stateR2.Count > 0) {
                                         optimalStates.Remove(stateY);
                                         optimalStates.Add(stateR1);
                                         optimalStates.Add(stateR2);
                                         if (queue.Contains(stateY)) {
                                             queue.Remove(stateY);
                                             queue.Add(stateR1);
                                             queue.Add(stateR2);
                                         } else {
                                             queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
                                         }
                                     }
                                 }
                             }
                         }
+                        // дополнительно разбиваем конечные состояния
+                        foreach (var final in optimalStates.Where(s => s.Overlaps(m_finalStates)).ToArray()) {
+                            optimalStates.Remove(final);
+                            foreach (var split in SplitFinalStates(final))
+                                optimalStates.Add(split);
+                        }
                         // карта получения оптимального состояния по соотвествующему ему простому состоянию
                         var nextState = 0;
                         foreach (var item in optimalStates) {
                             var id = nextState++;
                             foreach (var s in item)
                                 stateMap[s] = id;
                         }
                         // получаем минимальный алфавит
                         // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
                         // для этого используем алгоритм кластеризации, сначала
                         // считаем, что все символы не различимы
                         var minClasses = new HashSet<HashSet<int>>(setComparer);
                         var alphaQueue = new Queue<HashSet<int>>();
                         alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
                         // для всех состояний, будем проверять каждый класс на различимость,
                         // т.е. символы различимы, если они приводят к разным состояниям
                         for (int s = 0 ; s < optimalStates.Count; s++) {
                             var newQueue = new Queue<HashSet<int>>();
                             foreach (var A in alphaQueue) {
                                 // классы из одного символа делить бесполезно, переводим их сразу в
                                 // результирующий алфавит
                                 if (A.Count == 1) {
                                     minClasses.Add(A);
                                     continue;
                                 }
                                 // различаем классы символов, которые переводят в различные оптимальные состояния
                                 // optimalState -> alphaClass
                                 var classes = new Dictionary<int, HashSet<int>>();
                                 foreach (var term in A) {
                                     // ищем все переходы класса по символу term
                                     var s2 = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).DefaultIfEmpty(-1).First();
                                     HashSet<int> a2;
                                     if (!classes.TryGetValue(s2, out a2)) {
                                         a2 = new HashSet<int>();
                                         newQueue.Enqueue(a2);
                                         classes[s2] = a2;
                                     }
                                     a2.Add(term);
                                 }
                             }
                             if (newQueue.Count == 0)
                                 break;
                             alphaQueue = newQueue;
                         }
                         // после окончания работы алгоритма в очереди останутся минимальные различимые классы
                         // входных символов
                         foreach (var A in alphaQueue)
                             minClasses.Add(A);
                         // построение отображения алфавитов входных символов.
                         // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
                         // специальное значение, тогда сохраним минимальный класс,
                         // содержащий этот символ на томже месте.
                         var nextCls = 0;
                         foreach (var item in minClasses) {
                             if (nextCls == AutomatonConst.UNCLASSIFIED_INPUT)
                                 nextCls++;
                             // сохраняем DFAConst.UNCLASSIFIED_INPUT
                             var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++;
                             optimalDFA.AddSymbol(cls);
                             foreach (var a in item)
                                 alphabetMap[a] = cls;
                         }
                         // построение автомата
                         optimalDFA.SetInitialState(stateMap[m_initialState]);
                         foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
                             optimalDFA.MarkFinalState(sf);
                         foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
                             optimalDFA.Add(t);
                     }
                     protected string PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
                         Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
                         Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
                         var data = new List<string>();
                         data.Add("digraph dfa {");
                         foreach (var final in m_finalStates)
                             data.Add(String.Format("{0} [shape=box];",String.Join("", stateAlphabet.GetSymbols(final))));
                         foreach (var t in m_transitions)
                             data.Add(String.Format(
                                 "{0} -> {2} [label={1}];",
                                 String.Join("", stateAlphabet.GetSymbols(t.s1)),
                                 ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UNCLASSIFIED_INPUT ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
                                 String.Join("", stateAlphabet.GetSymbols(t.s2))
                             ));
                         data.Add("}");
                         return String.Join("\n", data);
                     }
                     static string ToLiteral(string input)
                     {
                         using (var writer = new StringWriter())
                         {
                             using (var provider = CodeDomProvider.CreateProvider("CSharp"))
                             {
                                 provider.GenerateCodeFromExpression(new CodePrimitiveExpression(input), writer, null);
                                 return writer.ToString();
                             }
                         }
                     }
                 }
             }

Implab/Automaton/RegularExpressions/RegularDFA.cs +2 -6

             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton.RegularExpressions {
                 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
                     readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
                     readonly IAlphabet<TInput> m_alphabet;
                     public RegularDFA(IAlphabet<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "aplhabet");
                         m_alphabet = alphabet;
                     }
                     public IAlphabet<TInput> InputAlphabet {
                         get {
                             return m_alphabet;
                         }
                     }
                     public void MarkFinalState(int s, TTag[] tags) {
                         MarkFinalState(s);
                         SetStateTag(s, tags);
                     }
                     public void SetStateTag(int s, TTag[] tags) {
                         Safe.ArgumentNotNull(tags, "tags");
                         m_tags[s] = tags;
                     }
                     public TTag[] GetStateTag(int s) {
                         TTag[] tags;
                         return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
                     }
                     public TTag[][] CreateTagTable() {
                         var table = new TTag[StateCount][];
                         foreach (var pair in m_tags)
                             table[pair.Key] = pair.Value;
                         return table;
                     }
                     /// <summary>
                     /// Optimize the specified alphabet.
                     /// </summary>
                     /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
                     public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "alphabet");
                         var dfa = new RegularDFA<TInput, TTag>(alphabet);
                         var alphaMap = new Dictionary<int,int>();
                         var stateMap = new Dictionary<int,int>();
                         Optimize(dfa, alphaMap, stateMap);
                         // mark tags in the new DFA
                         foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
                             dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
                         // make the alphabet for the new DFA
                         // skip all unclassified symbols
                         foreach (var pair in alphaMap.Where(x => x.Value != 0))
                             alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
-                        var orig = ToString();
-                        var opt = dfa.ToString();
                         return dfa;
                     }
-                    protected override IEnumerable<HashSet<int>> GroupFinalStates() {
+                    protected override IEnumerable<HashSet<int>> SplitFinalStates(IEnumerable<int> states) {
                         var arrayComparer = new CustomEqualityComparer<TTag[]>(
                             (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
                             x => x.Sum(it => x.GetHashCode())
                         );
-                        return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
+                        return states.GroupBy(x => m_tags[x] ?? new TTag[0], arrayComparer).Select(g => new HashSet<int>(g));
                     }
                     public override string ToString() {
                         var states = new MapAlphabet<string>(false, null);
                         for (int i = 0; i < StateCount; i++)
                             states.DefineSymbol(string.Format("s{0}", i), i);
                         return string.Format("//[RegularDFA {1} x {2}]\n{0}", PrintDFA(InputAlphabet, states),StateCount, AlphabetSize);
                     }
                 }
             }

Implab/Formats/JSON/JSONGrammar.cs +3 -1

             using System.Linq;
             using Implab.Automaton.RegularExpressions;
             using System;
             using Implab.Automaton;
             using Implab.Components;
             namespace Implab.Formats.JSON {
                 class JSONGrammar : Grammar<char> {
                     public enum TokenType {
                         None,
                         BeginObject,
                         EndObject,
                         BeginArray,
                         EndArray,
                         String,
                         Number,
                         Literal,
                         NameSeparator,
                         ValueSeparator,
+                        Whitespace,
                         StringBound,
                         EscapedChar,
                         UnescapedChar,
                         EscapedUnicode
                     }
                     static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
                     public static JSONGrammar Instance {
                         get { return _instance.Value; }
                     }
                     readonly ScannerContext<TokenType> m_jsonExpression;
                     readonly ScannerContext<TokenType> m_stringExpression;
                     readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
                     public JSONGrammar() {
                         DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
                         var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
                         var digit9 = SymbolRangeToken('1', '9');
                         var zero = SymbolToken('0');
                         var digit = zero.Or(digit9);
                         var dot = SymbolToken('.');
                         var minus = SymbolToken('-');
                         var sign = SymbolSetToken('-', '+');
                         var expSign = SymbolSetToken('e', 'E');
                         var letters = SymbolRangeToken('a', 'z');
                         var integer = zero.Or(digit9.Cat(digit.EClosure()));
                         var frac = dot.Cat(digit.Closure());
                         var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
                         var quote = SymbolToken('"');
                         var backSlash = SymbolToken('\\');
                         var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
                         var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
                         var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
                         var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
                         var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
                         var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
                         var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
                         var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
                         var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
                         var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
                         var literal = letters.Closure();
                         var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
                         var jsonExpression =
                             number.Tag(TokenType.Number)
                             .Or(literal.Tag(TokenType.Literal))
                             .Or(quote.Tag(TokenType.StringBound))
                             .Or(beginObject.Tag(TokenType.BeginObject))
                             .Or(endObject.Tag(TokenType.EndObject))
                             .Or(beginArray.Tag(TokenType.BeginArray))
                             .Or(endArray.Tag(TokenType.EndArray))
                             .Or(nameSep.Tag(TokenType.NameSeparator))
-                            .Or(valueSep.Tag(TokenType.ValueSeparator));
+                            .Or(valueSep.Tag(TokenType.ValueSeparator))
+                            .Or(SymbolSetToken('\n', '\r', '\t', ' ').Closure().Tag(TokenType.Whitespace));
                         var jsonStringExpression =
                             quote.Tag(TokenType.StringBound)
                             .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
                             .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
                             .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
                         m_jsonExpression = BuildScannerContext<TokenType>(jsonExpression);
                         m_stringExpression = BuildScannerContext<TokenType>(jsonStringExpression);
                     }
                     protected override IAlphabetBuilder<char> AlphabetBuilder {
                         get {
                             return m_defaultAlphabet;
                         }
                     }
                     public ScannerContext<TokenType> JsonExpression {
                         get {
                             return m_jsonExpression;
                         }
                     }
                     public ScannerContext<TokenType> JsonStringExpression {
                         get {
                             return m_stringExpression;
                         }
                     }
                     Token SymbolRangeToken(char start, char stop) {
                         return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
                     }
                     protected override IndexedAlphabetBase<char> CreateAlphabet() {
                         return new CharAlphabet();
                     }
                 }
             }

Implab/Formats/JSON/JSONScanner.cs +3 -1

             using System;
             using System.Globalization;
             using Implab.Automaton;
             using System.Text;
             using Implab.Components;
             using System.IO;
             namespace Implab.Formats.JSON {
                 /// <summary>
                 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
                 /// </summary>
                 public class JSONScanner : Disposable {
                     readonly StringBuilder m_builder = new StringBuilder();
                     readonly ScannerContext<JSONGrammar.TokenType> m_jsonContext = JSONGrammar.Instance.JsonExpression;
                     readonly ScannerContext<JSONGrammar.TokenType> m_stringContext = JSONGrammar.Instance.JsonStringExpression;
                     readonly TextScanner m_scanner;
                     /// <summary>
                     /// Создает новый экземпляр сканнера
                     /// </summary>
                     public JSONScanner(string text) {
                         Safe.ArgumentNotEmpty(text, "text");
                         m_scanner = new StringScanner(text);
                     }
                     public JSONScanner(TextReader reader, int bufferMax, int chunkSize) {
                         Safe.ArgumentNotNull(reader, "reader");
                         m_scanner = new ReaderScanner(reader, bufferMax, chunkSize);
                     }
                     public JSONScanner(TextReader reader) : this(reader, 1024*1024, 1024){
                     }
                     /// <summary>
                     /// Читает следующий лексический элемент из входных данных.
                     /// </summary>
                     /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
                     /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
                     /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
                     /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
                     /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
                     public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
                         JSONGrammar.TokenType[] tag;
-                        if (m_jsonContext.Execute(m_scanner, out tag)) {
+                        while (m_jsonContext.Execute(m_scanner, out tag)) {
                             switch (tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
                                     tokenValue = ReadString();
                                     tokenType = JsonTokenType.String;
                                     break;
                                 case JSONGrammar.TokenType.Number:
                                     tokenValue = Double.Parse(m_scanner.GetTokenValue(), CultureInfo.InvariantCulture);
                                     tokenType = JsonTokenType.Number;
                                     break;
+                                case JSONGrammar.TokenType.Whitespace:
+                                    continue;
                                 default:
                                     tokenType = (JsonTokenType)tag[0];
                                     tokenValue = m_scanner.GetTokenValue();
                                     break;
                             }
                             return true;
                         }
                         tokenValue = null;
                         tokenType = JsonTokenType.None;
                         return false;
                     }
                     string ReadString() {
                         int pos = 0;
                         var buf = new char[6]; // the buffer for unescaping chars
                         JSONGrammar.TokenType[] tag;
                         m_builder.Clear();
                         while (m_stringContext.Execute(m_scanner, out tag)) {
                             switch (tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
                                     return m_builder.ToString();
                                 case JSONGrammar.TokenType.UnescapedChar:
                                     m_scanner.CopyTokenTo(m_builder);
                                     break;
                                 case JSONGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
                                     m_scanner.CopyTokenTo(buf, 0);
                                     m_builder.Append(StringTranslator.TranslateHexUnicode(buf, 2));
                                     pos++;
                                     break;
                                 case JSONGrammar.TokenType.EscapedChar:  // \t - escape sequence
                                     m_scanner.CopyTokenTo(buf, 0);
                                     m_builder.Append(StringTranslator.TranslateEscapedChar(buf[1]));
                                     break;
                             }
                         }
                         throw new ParserException("Unexpected end of data");
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing)
                             Safe.Dispose(m_scanner);
                         base.Dispose(disposing);
                     }
                 }
             }

Implab/Formats/ReaderScanner.cs +1 -1

             using System;
             using System.IO;
             namespace Implab.Formats {
                 public class ReaderScanner: TextScanner {
-                    const int CHUNK_SIZE = 1024;
+                    const int CHUNK_SIZE = 1024*4;
                     const int BUFFER_MAX = CHUNK_SIZE*1024;
                     readonly TextReader m_reader;
                     public ReaderScanner(TextReader reader, int limit, int chunk) : base(limit, chunk) {
                         Safe.ArgumentNotNull(reader, "reader");
                         m_reader = reader;
                     }
                     public ReaderScanner(TextReader reader) : this(reader, BUFFER_MAX, CHUNK_SIZE) {
                     }
                     protected override int Read(char[] buffer, int offset, int size) {
                         return m_reader.Read(buffer, offset, size);
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing)
                             Safe.Dispose(m_reader);
                         base.Dispose(disposing);
                     }
                 }
             }

MonoPlay/MonoPlay.csproj +6 0

             <?xml version="1.0" encoding="utf-8"?>
             <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
               <PropertyGroup>
                 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
                 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
                 <ProductVersion>8.0.30703</ProductVersion>
                 <SchemaVersion>2.0</SchemaVersion>
                 <ProjectGuid>{15DD7123-D504-4627-8B4F-D00C7F04D033}</ProjectGuid>
                 <OutputType>Exe</OutputType>
                 <RootNamespace>MonoPlay</RootNamespace>
                 <AssemblyName>MonoPlay</AssemblyName>
                 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
                 <ReleaseVersion>0.2</ReleaseVersion>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>DEBUG;TRACE;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
                 <DebugType>full</DebugType>
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <ItemGroup>
                 <Reference Include="System" />
+                <Reference Include="System.Text.Json">
+                  <HintPath>..\packages\System.Text.Json.2.0.0.11\lib\net40\System.Text.Json.dll</HintPath>
+                </Reference>
               </ItemGroup>
               <ItemGroup>
                 <Compile Include="Program.cs" />
                 <Compile Include="Properties\AssemblyInfo.cs" />
               </ItemGroup>
               <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
               <ItemGroup>
                 <ProjectReference Include="..\Implab\Implab.csproj">
                   <Project>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</Project>
                   <Name>Implab</Name>
                 </ProjectReference>
               </ItemGroup>
+              <ItemGroup>
+                <None Include="packages.config" />
+              </ItemGroup>
             </Project>

MonoPlay/Program.cs +24 -16

             using System;
             using Implab;
             using System.Threading.Tasks;
+            using Implab.Formats.JSON;
+            using System.IO;
+            using System.Text.Json;
             namespace MonoPlay {
                 class MainClass {
                     public static void Main(string[] args) {
                         if (args == null)
                             throw new ArgumentNullException("args");
+                        int t1, t2;
-                        var t1 = Environment.TickCount;
-                        DoWork().GetAwaiter().GetResult();
+                        for (int i = 0; i < 2; i++) {
+                            t1 = Environment.TickCount;
+                            int elements =0;
+                            using (var reader = new JSONParser(File.OpenText("/home/sergey/temp/citylots.json"))) {
+                                while (reader.Read())
+                                    elements++;
+                            }
-                        var t2 = Environment.TickCount;
+                            t2 = Environment.TickCount;
-                        Console.WriteLine("done: {0} ms, {1:.00} Mb, {2} GC", t2 - t1, GC.GetTotalMemory(false) / (1024*1024), GC.CollectionCount(0) );
+                            Console.WriteLine("attempt {0} done: {1} ms, {2:.00} Mb, {3} GC, Elements: {4}",i+1, t2 - t1, GC.GetTotalMemory(false) / (1024*1024), GC.CollectionCount(0), elements );
                         }
-                    static IPromise<int> DoItem(int x) {
+                        Console.WriteLine("Syste.Text.Json");
-                        //return Promise<int>.FromResult(x + 1);
+                        var paraser = new JsonParser();
-                        var p = new Promise<int>();
+                        for (int i = 0; i < 2; i++) {
-                        p.Resolve(x+1);
+                            t1 = Environment.TickCount;
-                        return p;
+                            using (var reader = File.OpenText("/home/sergey/temp/citylots.json")) {
+                                paraser.Parse(reader);
                             }
-                    static async Task<int> DoWork() {
+                            t2 = Environment.TickCount;
-                        var c = 0;
+                            Console.WriteLine("attempt {0} done: {1} ms, {2:.00} Mb, {3} GC, ",i+1, t2 - t1, GC.GetTotalMemory(false) / (1024*1024), GC.CollectionCount(0));
-                        for (int i = 0; i < 10000000; i++)
-                            c = await DoItem(c);
-                        return c;
                     }
                 }
             }

packages/repositories.config 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages