implab/ImplabNet Commit - r182:76e8f2ba12b8

pretty print DFA, the minimization is still buggy

cin -

r182:76e8f2ba12b8 ref20160224

parent child

Context file:

r182:76e8f2ba12b8

Collapse all files

Implab.Test/Implab.Format.Test/Implab.Format.Test.csproj +8 -1

             <?xml version="1.0" encoding="utf-8"?>
             <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
               <PropertyGroup>
                 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
                 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
                 <ProductVersion>8.0.30703</ProductVersion>
                 <SchemaVersion>2.0</SchemaVersion>
                 <ProjectGuid>{4D364996-7ECD-4193-8F90-F223FFEA49DA}</ProjectGuid>
                 <OutputType>Library</OutputType>
                 <RootNamespace>Implab.Format.Test</RootNamespace>
                 <AssemblyName>Implab.Format.Test</AssemblyName>
                 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+                <ReleaseVersion>0.2</ReleaseVersion>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>DEBUG;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
                 <DebugType>full</DebugType>
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <ItemGroup>
                 <Reference Include="System" />
                 <Reference Include="nunit.framework">
-                  <HintPath>..\..\packages\NUnit.3.0.1\lib\net45\nunit.framework.dll</HintPath>
+                  <HintPath>..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll</HintPath>
                 </Reference>
               </ItemGroup>
               <ItemGroup>
                 <Compile Include="JsonTests.cs" />
               </ItemGroup>
               <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
               <ItemGroup>
+                <ProjectReference Include="..\..\Implab\Implab.csproj">
+                  <Project>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</Project>
+                  <Name>Implab</Name>
+                </ProjectReference>
+              </ItemGroup>
+              <ItemGroup>
                 <None Include="packages.config" />
               </ItemGroup>
             </Project>

Implab.Test/Implab.Format.Test/JsonTests.cs +41 -3

@@ -1,12 +1,50
1	using NUnit.Framework;	1	using NUnit.Framework;
2	using System;	2	using System;
		3	using Implab.Formats.JSON;
3		4
4	namespace Implab.Format.Test {	5	namespace Implab.Format.Test {
5	[TestFixture()]	6	[TestFixture]
6	public class JsonTests {	7	public class JsonTests {
7	[Test()]	8	[Test]
8	public void Test~~Case~~() {	9	public void TestScannerValidTokens() {
		10	var scanner = new JSONScanner(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:");
		11
		12	Tuple<JsonTokenType,object>[] expexted = new [] {
		13	new Tuple<JsonTokenType,object>(JsonTokenType.Number, 9123d),
		14	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		15	new Tuple<JsonTokenType,object>(JsonTokenType.Number, -123d ),
		16	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		17	new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0d ),
		18	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		19	new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0.1d ),
		20	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		21	new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.2d ),
		22	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		23	new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.1e3d ),
		24	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		25	new Tuple<JsonTokenType,object>(JsonTokenType.Number, 1.3E-3d ),
		26	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		27	new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n text" ),
		28	new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", " ),
		29	new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal" ),
		30	new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, " [" ),
		31	new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, "]" ),
		32	new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, "{" ),
		33	new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, "}" ),
		34	new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, ":" )
		35	};
		36
		37	object value;
		38	JsonTokenType tokenType;
		39	for (var i = 0; i < expexted.Length; i++) {
		40
		41	Assert.IsTrue(scanner.ReadToken(out value, out tokenType));
		42	Assert.AreEqual(expexted[i].Item1, tokenType);
		43	Assert.AreEqual(expexted[i].Item2, value);
		44	}
		45
		46	Assert.IsFalse(scanner.ReadToken(out value, out tokenType));
9	}	47	}
10	}	48	}
11	}	49	}
12		50

Implab.Test/Implab.Format.Test/packages.config +1 -1

             <?xml version="1.0" encoding="utf-8"?>
             <packages>
-              <package id="NUnit" version="3.0.1" targetFramework="net45" />
+              <package id="NUnit" version="2.6.4" targetFramework="net45" />
             </packages>

Implab/Automaton/DFATable.cs +44 -17

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             using System.Diagnostics;
+            using System.IO;
+            using System.CodeDom.Compiler;
+            using System.CodeDom;
             namespace Implab.Automaton {
                 public class DFATable : IDFATableBuilder {
                     int m_stateCount;
                     int m_symbolCount;
                     int m_initialState;
                     readonly HashSet<int> m_finalStates = new HashSet<int>();
                     readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
                     #region IDFADefinition implementation
                     public bool IsFinalState(int s) {
                         Safe.ArgumentInRange(s, 0, m_stateCount, "s");
                         return m_finalStates.Contains(s);
                     }
                     public IEnumerable<int> FinalStates {
                         get {
                             return m_finalStates;
                         }
                     }
                     public int StateCount {
                         get { return m_stateCount; }
                     }
                     public int AlphabetSize {
                         get { return m_symbolCount; }
                     }
                     public int InitialState {
                         get { return m_initialState; }
                     }
                     #endregion
                     public void SetInitialState(int s) {
                         Safe.ArgumentAssert(s >= 0, "s");
                         m_stateCount = Math.Max(m_stateCount, s + 1);
                         m_initialState = s;
                     }
                     public void MarkFinalState(int state) {
                         m_stateCount = Math.Max(m_stateCount, state + 1);
                         m_finalStates.Add(state);
                     }
                     public void Add(AutomatonTransition item) {
                         Safe.ArgumentAssert(item.s1 >= 0, "item");
                         Safe.ArgumentAssert(item.s2 >= 0, "item");
                         Safe.ArgumentAssert(item.edge >= 0, "item");
                         m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
                         m_symbolCount = Math.Max(m_symbolCount, item.edge + 1);
                         m_transitions.Add(item);
                     }
                     public void Clear() {
                         m_stateCount = 0;
                         m_symbolCount = 0;
                         m_finalStates.Clear();
                         m_transitions.Clear();
                     }
                     public bool Contains(AutomatonTransition item) {
                         return m_transitions.Contains(item);
                     }
                     public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
                         m_transitions.CopyTo(array, arrayIndex);
                     }
                     public bool Remove(AutomatonTransition item) {
                         return m_transitions.Remove(item);
                     }
                     public int Count {
                         get {
                             return m_transitions.Count;
                         }
                     }
                     public bool IsReadOnly {
                         get {
                             return false;
                         }
                     }
                     public IEnumerator<AutomatonTransition> GetEnumerator() {
                         return m_transitions.GetEnumerator();
                     }
                     System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
                         return GetEnumerator();
                     }
+                    public void AddSymbol(int symbol) {
+                        Safe.ArgumentAssert(symbol >= 0, "symbol");
+                        m_symbolCount = Math.Max(symbol + 1, m_symbolCount);
+                    }
                     public int[,] CreateTransitionTable() {
                         var table = new int[StateCount,AlphabetSize];
                         for (int i = 0; i < StateCount; i++)
                             for (int j = 0; j < AlphabetSize; j++)
                                 table[i, j] = AutomatonConst.UNREACHABLE_STATE;
                         foreach (var t in this)
                             table[t.s1,t.edge] = t.s2;
                         return table;
                     }
                     public bool[] CreateFinalStateTable() {
                         var table = new bool[StateCount];
                         foreach (var s in FinalStates)
                             table[s] = true;
                         return table;
                     }
                     /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
                     /// <remarks>
                     /// В процессе построения минимального автомата требуется разделить множество состояний,
                     /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
                     /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
                     /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
                     /// </remarks>
                     /// <returns>The final states.</returns>
                     protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
                         return new HashSet<int>[] { m_finalStates };
                     }
                     protected void Optimize(
                         IDFATableBuilder optimalDFA,
                         IDictionary<int,int> alphabetMap,
                         IDictionary<int,int> stateMap
                     ) {
                         Safe.ArgumentNotNull(optimalDFA, "dfa");
                         Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
                         Safe.ArgumentNotNull(stateMap, "stateMap");
                         var setComparer = new CustomEqualityComparer<HashSet<int>>(
                             (x, y) => x.SetEquals(y),
                             s => s.Sum(x => x.GetHashCode())
                         );
                         var optimalStates = new HashSet<HashSet<int>>(setComparer);
                         var queue = new HashSet<HashSet<int>>(setComparer);
                         // получаем конечные состояния, сгруппированные по маркерам
                         optimalStates.UnionWith(
                             GroupFinalStates()
                         );
                         var state = new HashSet<int>(
                             Enumerable
-                            .Range(0, m_stateCount - 1)
+                            .Range(0, m_stateCount)
                             .Where(i => !m_finalStates.Contains(i))
                         );
                         optimalStates.Add(state);
                         queue.Add(state);
                         var rmap = m_transitions
                             .GroupBy(t => t.s2)
                             .ToDictionary(
                                 g => g.Key, // s2
                                 g => g.ToLookup(t => t.edge, t => t.s1)//.ToDictionary(p => p.Key)
                             );
                         while (queue.Count > 0) {
                             var stateA = queue.First();
                             queue.Remove(stateA);
                             for (int c = 0; c < m_symbolCount; c++) {
                                 var stateX = new HashSet<int>();
-                                foreach(var a in stateA.Where(rmap.ContainsKey))
+                                //foreach(var a in stateA.Where(rmap.ContainsKey))
-                                    stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
+                                //    stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
-                                foreach (var stateY in optimalStates.ToArray()) {
+                                stateX.UnionWith(m_transitions.Where(t => stateA.Contains(t.s2) && t.edge == c).Select(t => t.s1));
+                                var tmp = optimalStates.ToArray();
+                                foreach (var stateY in tmp) {
                                     if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
                                         var stateR1 = new HashSet<int>(stateY);
                                         var stateR2 = new HashSet<int>(stateY);
                                         stateR1.IntersectWith(stateX);
                                         stateR2.ExceptWith(stateX);
                                         optimalStates.Remove(stateY);
                                         optimalStates.Add(stateR1);
                                         optimalStates.Add(stateR2);
                                         if (queue.Contains(stateY)) {
                                             queue.Remove(stateY);
                                             queue.Add(stateR1);
                                             queue.Add(stateR2);
                                         } else {
                                             queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
                                         }
                                     }
                                 }
                             }
                         }
                         // карта получения оптимального состояния по соотвествующему ему простому состоянию
                         var nextState = 0;
                         foreach (var item in optimalStates) {
                             var id = nextState++;
                             foreach (var s in item)
                                 stateMap[s] = id;
                         }
                         // получаем минимальный алфавит
                         // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
                         // для этого используем алгоритм кластеризации, сначала
                         // считаем, что все символы не различимы
                         var minClasses = new HashSet<HashSet<int>>(setComparer);
                         var alphaQueue = new Queue<HashSet<int>>();
                         alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
                         // для всех состояний, будем проверять каждый класс на различимость,
                         // т.е. символы различимы, если они приводят к разным состояниям
                         for (int s = 0 ; s < optimalStates.Count; s++) {
                             var newQueue = new Queue<HashSet<int>>();
                             foreach (var A in alphaQueue) {
                                 // классы из одного символа делить бесполезно, переводим их сразу в
                                 // результирующий алфавит
                                 if (A.Count == 1) {
                                     minClasses.Add(A);
                                     continue;
                                 }
                                 // различаем классы символов, которые переводят в различные оптимальные состояния
                                 // optimalState -> alphaClass
                                 var classes = new Dictionary<int, HashSet<int>>();
                                 foreach (var term in A) {
                                     // ищем все переходы класса по символу term
-                                    var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray();
+                                    var s2 = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).DefaultIfEmpty(-1).First();
-                                    Debug.Assert(res.Length <= 1);
-                                    var s2 = res.Length > 0 ? res[0] : -1;
                                     HashSet<int> a2;
                                     if (!classes.TryGetValue(s2, out a2)) {
                                         a2 = new HashSet<int>();
                                         newQueue.Enqueue(a2);
                                         classes[s2] = a2;
                                     }
                                     a2.Add(term);
                                 }
                             }
                             if (newQueue.Count == 0)
                                 break;
                             alphaQueue = newQueue;
                         }
                         // после окончания работы алгоритма в очереди останутся минимальные различимые классы
                         // входных символов
                         foreach (var A in alphaQueue)
                             minClasses.Add(A);
                         // построение отображения алфавитов входных символов.
                         // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
                         // специальное значение, тогда сохраним минимальный класс,
                         // содержащий этот символ на томже месте.
                         var nextCls = 0;
                         foreach (var item in minClasses) {
                             if (nextCls == AutomatonConst.UNCLASSIFIED_INPUT)
                                 nextCls++;
                             // сохраняем DFAConst.UNCLASSIFIED_INPUT
                             var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++;
+                            optimalDFA.AddSymbol(cls);
                             foreach (var a in item)
                                 alphabetMap[a] = cls;
                         }
                         // построение автомата
                         optimalDFA.SetInitialState(stateMap[m_initialState]);
                         foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
                             optimalDFA.MarkFinalState(sf);
                         foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
                             optimalDFA.Add(t);
                     }
-                    protected void PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
+                    protected string PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
                         Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
                         Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
+                        var data = new List<string>();
+                        data.Add("digraph dfa {");
+                        foreach (var final in m_finalStates)
+                            data.Add(String.Format("{0} [shape=box];",String.Join("", stateAlphabet.GetSymbols(final))));
                         foreach (var t in m_transitions)
-                            Console.WriteLine(
+                            data.Add(String.Format(
-                                "[{0}] -{{{1}}}-> [{2}]{3}",
+                                "{0} -> {2} [label={1}];",
-                                String.Join(",", stateAlphabet.GetSymbols(t.s1)),
+                                String.Join("", stateAlphabet.GetSymbols(t.s1)),
-                                String.Join("", inputAlphabet.GetSymbols(t.edge)),
+                                ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UNCLASSIFIED_INPUT ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
-                                String.Join(",", stateAlphabet.GetSymbols(t.s2)),
+                                String.Join("", stateAlphabet.GetSymbols(t.s2))
-                                m_finalStates.Contains(t.s2) ? "$" : ""
+                            ));
-                            );
+                        data.Add("}");
+                        return String.Join("\n", data);
                     }
+                    static string ToLiteral(string input)
+                    {
+                        using (var writer = new StringWriter())
+                        {
+                            using (var provider = CodeDomProvider.CreateProvider("CSharp"))
+                            {
+                                provider.GenerateCodeFromExpression(new CodePrimitiveExpression(input), writer, null);
+                                return writer.ToString();
                             }
                         }
+                    }
+                }
+            }

Implab/Automaton/IDFATableBuilder.cs +11 0

             using System;
             using System.Collections.Generic;
             namespace Implab.Automaton {
                 public interface IDFATableBuilder : IDFATable, ICollection<AutomatonTransition> {
                     /// <summary>
                     /// Marks the state as final.
                     /// </summary>
                     /// <param name="state">State.</param>
                     void MarkFinalState(int state);
                     void SetInitialState(int s);
+                    /// <summary>
+                    /// Increases if needed the input alphabet size to hold the specified symbol.
+                    /// </summary>
+                    /// <remarks>
+                    /// <code>
+                    /// AlphabetSize = Math.Max(AlphabetSize, symbol + 1)
+                    /// </code>
+                    /// </remarks>
+                    /// <param name="symbol">Symbol.</param>
+                    void AddSymbol(int symbol);
                 }
             }

Implab/Automaton/RegularExpressions/RegularDFA.cs +12 0

             using System.Collections.Generic;
             using System.Linq;
             namespace Implab.Automaton.RegularExpressions {
                 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
                     readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
                     readonly IAlphabet<TInput> m_alphabet;
                     public RegularDFA(IAlphabet<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "aplhabet");
                         m_alphabet = alphabet;
                     }
                     public IAlphabet<TInput> InputAlphabet {
                         get {
                             return m_alphabet;
                         }
                     }
                     public void MarkFinalState(int s, TTag[] tags) {
                         MarkFinalState(s);
                         SetStateTag(s, tags);
                     }
                     public void SetStateTag(int s, TTag[] tags) {
                         Safe.ArgumentNotNull(tags, "tags");
                         m_tags[s] = tags;
                     }
                     public TTag[] GetStateTag(int s) {
                         TTag[] tags;
                         return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
                     }
                     public TTag[][] CreateTagTable() {
                         var table = new TTag[StateCount][];
                         foreach (var pair in m_tags)
                             table[pair.Key] = pair.Value;
                         return table;
                     }
                     /// <summary>
                     /// Optimize the specified alphabet.
                     /// </summary>
                     /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
                     public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
                         Safe.ArgumentNotNull(alphabet, "alphabet");
                         var dfa = new RegularDFA<TInput, TTag>(alphabet);
                         var alphaMap = new Dictionary<int,int>();
                         var stateMap = new Dictionary<int,int>();
                         Optimize(dfa, alphaMap, stateMap);
                         // mark tags in the new DFA
                         foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
                             dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
                         // make the alphabet for the new DFA
                         // skip all unclassified symbols
                         foreach (var pair in alphaMap.Where(x => x.Value != 0))
                             alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
+                        var orig = ToString();
+                        var opt = dfa.ToString();
                         return dfa;
                     }
                     protected override IEnumerable<HashSet<int>> GroupFinalStates() {
                         var arrayComparer = new CustomEqualityComparer<TTag[]>(
                             (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
                             x => x.Sum(it => x.GetHashCode())
                         );
                         return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
                     }
+                    public override string ToString() {
+                        var states = new MapAlphabet<string>(false, null);
+                        for (int i = 0; i < StateCount; i++)
+                            states.DefineSymbol(string.Format("s{0}", i), i);
+                        return string.Format("//[RegularDFA {1} x {2}]\n{0}", PrintDFA(InputAlphabet, states),StateCount, AlphabetSize);
+                    }
                 }
             }

Implab/Components/LazyAndWeak.cs +2 -1

             using System;
             using System.Threading;
             namespace Implab.Components {
                 /// <summary>
                 /// Creates an instace on-demand and allows it to be garbage collected.
                 /// </summary>
                 /// <remarks>
                 /// Usefull when dealing with memory-intensive objects which are frequently used.
-                /// This class is similar to <see cref="ObjectPool{T}"/> except is a singleton.
+                /// This class is similar to <see cref="ObjectPool{T}"/> except it is a singleton.
                 /// </remarks>
                 public class LazyAndWeak<T> where T : class {
                     readonly Func<T> m_factory;
                     readonly object m_lock;
                     WeakReference m_reference;
                     public LazyAndWeak(Func<T> factory, bool useLock) {
                         Safe.ArgumentNotNull(factory, "factory");
                         m_factory = factory;
                         m_lock = useLock ? new object() : null;
                     }
                     public LazyAndWeak(Func<T> factory) : this(factory, false) {
                     }
                     public T Value {
                         get {
                             while (true) {
                                 var weak = m_reference;
                                 T value;
                                 if (weak != null) {
                                     value = weak.Target as T;
                                     if (value != null)
                                         return value;
                                 }
                                 if (m_lock == null) {
                                     value = m_factory();
                                     if (Interlocked.CompareExchange(ref m_reference, new WeakReference(value), weak) == weak)
                                         return value;
                                 } else {
                                     lock (m_lock) {
                                         // double check
+                                        weak = m_reference;
                                         if (weak != null) {
                                             value = weak.Target as T;
                                             if (value != null)
                                                 return value;
                                         }
                                         // we are safe to write
                                         value = m_factory();
                                         m_reference = new WeakReference(value);
                                         return value;
                                     }
                                 }
                             }
                         }
                     }
                 }
             }

Implab/Formats/JSON/JSONGrammar.cs +1 -1

             using System.Linq;
             using Implab.Automaton.RegularExpressions;
             using System;
             using Implab.Automaton;
             using Implab.Components;
             namespace Implab.Formats.JSON {
                 class JSONGrammar : Grammar<char> {
                     public enum TokenType {
                         None,
                         BeginObject,
                         EndObject,
                         BeginArray,
                         EndArray,
                         String,
                         Number,
                         Literal,
                         NameSeparator,
                         ValueSeparator,
                         StringBound,
                         EscapedChar,
                         UnescapedChar,
                         EscapedUnicode
                     }
                     static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
                     public static JSONGrammar Instance {
                         get { return _instance.Value; }
                     }
                     readonly ScannerContext<TokenType> m_jsonExpression;
                     readonly ScannerContext<TokenType> m_stringExpression;
                     readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
                     public JSONGrammar() {
                         DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
                         var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
                         var digit9 = SymbolRangeToken('1', '9');
                         var zero = SymbolToken('0');
                         var digit = zero.Or(digit9);
                         var dot = SymbolToken('.');
                         var minus = SymbolToken('-');
                         var sign = SymbolSetToken('-', '+');
                         var expSign = SymbolSetToken('e', 'E');
                         var letters = SymbolRangeToken('a', 'z');
                         var integer = zero.Or(digit9.Cat(digit.EClosure()));
                         var frac = dot.Cat(digit.Closure());
                         var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
                         var quote = SymbolToken('"');
                         var backSlash = SymbolToken('\\');
                         var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
                         var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
                         var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
                         var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
                         var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
                         var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
                         var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
                         var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
                         var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
                         var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
                         var literal = letters.Closure();
                         var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
                         var jsonExpression =
                             number.Tag(TokenType.Number)
                             .Or(literal.Tag(TokenType.Literal))
                             .Or(quote.Tag(TokenType.StringBound))
                             .Or(beginObject.Tag(TokenType.BeginObject))
                             .Or(endObject.Tag(TokenType.EndObject))
                             .Or(beginArray.Tag(TokenType.BeginArray))
                             .Or(endArray.Tag(TokenType.EndArray))
                             .Or(nameSep.Tag(TokenType.NameSeparator))
                             .Or(valueSep.Tag(TokenType.ValueSeparator));
                         var jsonStringExpression =
                             quote.Tag(TokenType.StringBound)
                             .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
                             .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
                             .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
                         m_jsonExpression = BuildScannerContext<TokenType>(jsonExpression);
                         m_stringExpression = BuildScannerContext<TokenType>(jsonStringExpression);
                     }
                     protected override IAlphabetBuilder<char> AlphabetBuilder {
                         get {
                             return m_defaultAlphabet;
                         }
                     }
                     public ScannerContext<TokenType> JsonExpression {
                         get {
                             return m_jsonExpression;
                         }
                     }
                     public ScannerContext<TokenType> JsonStringExpression {
                         get {
                             return m_stringExpression;
                         }
                     }
                     Token SymbolRangeToken(char start, char stop) {
-                        return SymbolToken(Enumerable.Range(start,stop - start).Select(x => (char)x));
+                        return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
                     }
                     protected override IndexedAlphabetBase<char> CreateAlphabet() {
                         return new CharAlphabet();
                     }
                 }
             }

Implab/Formats/StringScanner.cs +5 -13

             using System;
             namespace Implab.Formats {
                 public class StringScanner: TextScanner {
                     const int CHUNK_SIZE = 1024;
-                    readonly string m_text;
+                    public StringScanner(string text) : base(null) {
-                    int m_pos;
+                        Safe.ArgumentNotNull(text, "text");
+                        var data = text.ToCharArray();
-                    public StringScanner(string text) : base(text.Length, text.Length < CHUNK_SIZE ? text.Length : CHUNK_SIZE) {
+                        Feed(data, 0, data.Length);
-                        m_text = text;
-                        Feed();
                     }
                     protected override int Read(char[] buffer, int offset, int size) {
-                        var actual = size + m_pos > m_text.Length ? m_text.Length - m_pos : size;
+                        return 0;
-                        m_text.CopyTo(m_pos,buffer,offset, actual);
-                        m_pos += actual;
-                        return actual;
                     }
                 }
             }

Implab/Formats/TextScanner.cs +5 -10

             using System;
             using Implab.Components;
             using System.Diagnostics;
             using Implab.Automaton;
             using System.Text;
             namespace Implab.Formats {
                 public abstract class TextScanner : Disposable {
                     readonly int m_bufferMax;
                     readonly int m_chunkSize;
                     char[] m_buffer;
                     int m_bufferOffset;
                     int m_bufferSize;
                     int m_tokenOffset;
                     int m_tokenLength;
                     /// <summary>
                     /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner"/> class.
                     /// </summary>
                     /// <param name="bufferMax">Buffer max.</param>
                     /// <param name="chunkSize">Chunk size.</param>
                     protected TextScanner(int bufferMax, int chunkSize) {
                         Debug.Assert(m_chunkSize <= m_bufferMax);
                         m_bufferMax = bufferMax;
                         m_chunkSize = chunkSize;
                     }
                     /// <summary>
                     /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner"/> class.
                     /// </summary>
                     /// <param name="buffer">Buffer.</param>
                     protected TextScanner(char[] buffer) {
                         if (buffer != null) {
                             m_buffer = buffer;
                             m_bufferSize = buffer.Length;
                         }
                     }
                     /// <summary>
                     /// (hungry) Reads the next token.
                     /// </summary>
                     /// <returns><c>true</c>, if token internal was read, <c>false</c> if there is no more tokens in the stream.</returns>
                     /// <param name="dfa">The transition map for the automaton</param>
                     /// <param name="final">Final states of the automaton.</param>
                     /// <param name="tags">Tags.</param>
                     /// <param name="state">The initial state for the automaton.</param>
                     /// <param name="alphabet"></param>
                     /// <param name = "tag"></param>
                     internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
                         m_tokenLength = 0;
                         tag = null;
                         var maxSymbol = alphabet.Length - 1;
+                        int next;
                         do {
                             // after the next chunk is read the offset in the buffer may change
                             int pos = m_bufferOffset + m_tokenLength;
+                            next = state;
                             while (pos < m_bufferSize) {
                                 var ch = m_buffer[pos];
-                                try {
+                                next = dfa[next, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
-                                var next = dfa[state, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
                                 if (next == AutomatonConst.UNREACHABLE_STATE)
                                     break;
                                 state = next;
-                                }catch {
-                                    throw;
                                 pos++;
                             }
                             m_tokenLength = pos - m_bufferOffset;
-                        } while (state != AutomatonConst.UNREACHABLE_STATE && Feed());
+                        } while (next != AutomatonConst.UNREACHABLE_STATE && Feed());
                         m_tokenOffset = m_bufferOffset;
                         m_bufferOffset += m_tokenLength;
                         if (final[state]) {
                             tag = tags[state];
                             return true;
                         }
                         if (m_bufferOffset == m_bufferSize) {
                             if (m_tokenLength == 0) //EOF
                                     return false;
                             throw new ParserException();
                         }
                         throw new ParserException(String.Format("Unexpected symbol '{0}'", m_buffer[m_bufferOffset]));
                     }
                     protected void Feed(char[] buffer, int offset, int length) {
                         m_buffer = buffer;
                         m_bufferOffset = offset;
                         m_bufferSize = offset + length;
                     }
                     protected bool Feed() {
                         if (m_chunkSize <= 0)
                             return false;
                         if (m_buffer != null) {
                             var free = m_buffer.Length - m_bufferSize;
                             if (free < m_chunkSize) {
                                 free += m_chunkSize;
                                 var used = m_bufferSize - m_bufferOffset;
                                 var size = used + free;
                                 if (size > m_bufferMax)
                                     throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax / 1024));
                                 var temp = new char[size];
                                 var read = Read(temp, used, m_chunkSize);
                                 if (read == 0)
                                     return false;
                                 Array.Copy(m_buffer, m_bufferOffset, temp, 0, used);
                                 m_bufferOffset = 0;
                                 m_bufferSize = used + read;
                                 m_buffer = temp;
                             } else {
                                 var read = Read(m_buffer, m_bufferSize, m_chunkSize);
                                 if (read == 0)
                                     return false;
                                 m_bufferSize += m_chunkSize;
                             }
                             return true;
                         } else {
                             Debug.Assert(m_bufferOffset == 0);
                             m_buffer = new char[m_chunkSize];
                             m_bufferSize = Read(m_buffer, 0, m_chunkSize);
                             return (m_bufferSize != 0);
                         }
                     }
                     protected abstract int Read(char[] buffer, int offset, int size);
                     public string GetTokenValue() {
                         return new String(m_buffer, m_tokenOffset, m_tokenLength);
                     }
                     public void CopyTokenTo(char[] buffer, int offset) {
-                        m_buffer.CopyTo(buffer, offset);
+                        Array.Copy(m_buffer, m_tokenOffset,buffer, offset, m_tokenLength);
                     }
                     public void CopyTokenTo(StringBuilder sb) {
                         sb.Append(m_buffer, m_tokenOffset, m_tokenLength);
                     }
                 }
             }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages