diff --git a/Implab.Test/Implab.Format.Test/Implab.Format.Test.csproj b/Implab.Test/Implab.Format.Test/Implab.Format.Test.csproj --- a/Implab.Test/Implab.Format.Test/Implab.Format.Test.csproj +++ b/Implab.Test/Implab.Format.Test/Implab.Format.Test.csproj @@ -10,6 +10,7 @@ Implab.Format.Test Implab.Format.Test v4.5 + 0.2 true @@ -32,7 +33,7 @@ - ..\..\packages\NUnit.3.0.1\lib\net45\nunit.framework.dll + ..\..\packages\NUnit.2.6.4\lib\nunit.framework.dll @@ -40,6 +41,12 @@ + + {F550F1F8-8746-4AD0-9614-855F4C4B7F05} + Implab + + + \ No newline at end of file diff --git a/Implab.Test/Implab.Format.Test/JsonTests.cs b/Implab.Test/Implab.Format.Test/JsonTests.cs --- a/Implab.Test/Implab.Format.Test/JsonTests.cs +++ b/Implab.Test/Implab.Format.Test/JsonTests.cs @@ -1,11 +1,49 @@ using NUnit.Framework; using System; +using Implab.Formats.JSON; namespace Implab.Format.Test { - [TestFixture()] + [TestFixture] public class JsonTests { - [Test()] - public void TestCase() { + [Test] + public void TestScannerValidTokens() { + var scanner = new JSONScanner(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:"); + + Tuple[] expexted = new [] { + new Tuple(JsonTokenType.Number, 9123d), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, -123d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, 0d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, 0.1d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, -0.2d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, -0.1e3d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Number, 1.3E-3d ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.String, "some \t\n text" ), + new Tuple(JsonTokenType.ValueSeparator, ", " ), + new Tuple(JsonTokenType.Literal, "literal" ), + new Tuple(JsonTokenType.BeginArray, " [" ), + new Tuple(JsonTokenType.EndArray, "]" ), + new Tuple(JsonTokenType.BeginObject, "{" ), + new Tuple(JsonTokenType.EndObject, "}" ), + new Tuple(JsonTokenType.NameSeparator, ":" ) + }; + + object value; + JsonTokenType tokenType; + for (var i = 0; i < expexted.Length; i++) { + + Assert.IsTrue(scanner.ReadToken(out value, out tokenType)); + Assert.AreEqual(expexted[i].Item1, tokenType); + Assert.AreEqual(expexted[i].Item2, value); + } + + Assert.IsFalse(scanner.ReadToken(out value, out tokenType)); } } } diff --git a/Implab.Test/Implab.Format.Test/packages.config b/Implab.Test/Implab.Format.Test/packages.config --- a/Implab.Test/Implab.Format.Test/packages.config +++ b/Implab.Test/Implab.Format.Test/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/Implab/Automaton/DFATable.cs b/Implab/Automaton/DFATable.cs --- a/Implab/Automaton/DFATable.cs +++ b/Implab/Automaton/DFATable.cs @@ -3,6 +3,9 @@ using System; using System.Collections.Generic; using System.Linq; using System.Diagnostics; +using System.IO; +using System.CodeDom.Compiler; +using System.CodeDom; namespace Implab.Automaton { public class DFATable : IDFATableBuilder { @@ -103,6 +106,11 @@ namespace Implab.Automaton { return GetEnumerator(); } + public void AddSymbol(int symbol) { + Safe.ArgumentAssert(symbol >= 0, "symbol"); + m_symbolCount = Math.Max(symbol + 1, m_symbolCount); + } + public int[,] CreateTransitionTable() { var table = new int[StateCount,AlphabetSize]; @@ -162,7 +170,7 @@ namespace Implab.Automaton { var state = new HashSet( Enumerable - .Range(0, m_stateCount - 1) + .Range(0, m_stateCount) .Where(i => !m_finalStates.Contains(i)) ); @@ -182,10 +190,13 @@ namespace Implab.Automaton { for (int c = 0; c < m_symbolCount; c++) { var stateX = new HashSet(); - foreach(var a in stateA.Where(rmap.ContainsKey)) - stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a' + //foreach(var a in stateA.Where(rmap.ContainsKey)) + // stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a' - foreach (var stateY in optimalStates.ToArray()) { + stateX.UnionWith(m_transitions.Where(t => stateA.Contains(t.s2) && t.edge == c).Select(t => t.s1)); + + var tmp = optimalStates.ToArray(); + foreach (var stateY in tmp) { if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) { var stateR1 = new HashSet(stateY); var stateR2 = new HashSet(stateY); @@ -245,12 +256,8 @@ namespace Implab.Automaton { foreach (var term in A) { // ищем все переходы класса по символу term - var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray(); + var s2 = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).DefaultIfEmpty(-1).First(); - Debug.Assert(res.Length <= 1); - - var s2 = res.Length > 0 ? res[0] : -1; - HashSet a2; if (!classes.TryGetValue(s2, out a2)) { a2 = new HashSet(); @@ -283,6 +290,7 @@ namespace Implab.Automaton { // сохраняем DFAConst.UNCLASSIFIED_INPUT var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++; + optimalDFA.AddSymbol(cls); foreach (var a in item) alphabetMap[a] = cls; @@ -298,19 +306,38 @@ namespace Implab.Automaton { optimalDFA.Add(t); } - protected void PrintDFA(IAlphabet inputAlphabet, IAlphabet stateAlphabet) { + protected string PrintDFA(IAlphabet inputAlphabet, IAlphabet stateAlphabet) { Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet"); Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet"); - foreach(var t in m_transitions) - Console.WriteLine( - "[{0}] -{{{1}}}-> [{2}]{3}", - String.Join(",", stateAlphabet.GetSymbols(t.s1)), - String.Join("", inputAlphabet.GetSymbols(t.edge)), - String.Join(",", stateAlphabet.GetSymbols(t.s2)), - m_finalStates.Contains(t.s2) ? "$" : "" - ); + var data = new List(); + + data.Add("digraph dfa {"); + + foreach (var final in m_finalStates) + data.Add(String.Format("{0} [shape=box];",String.Join("", stateAlphabet.GetSymbols(final)))); + + foreach (var t in m_transitions) + data.Add(String.Format( + "{0} -> {2} [label={1}];", + String.Join("", stateAlphabet.GetSymbols(t.s1)), + ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UNCLASSIFIED_INPUT ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))), + String.Join("", stateAlphabet.GetSymbols(t.s2)) + )); + data.Add("}"); + return String.Join("\n", data); } + static string ToLiteral(string input) + { + using (var writer = new StringWriter()) + { + using (var provider = CodeDomProvider.CreateProvider("CSharp")) + { + provider.GenerateCodeFromExpression(new CodePrimitiveExpression(input), writer, null); + return writer.ToString(); + } + } + } } } diff --git a/Implab/Automaton/IDFATableBuilder.cs b/Implab/Automaton/IDFATableBuilder.cs --- a/Implab/Automaton/IDFATableBuilder.cs +++ b/Implab/Automaton/IDFATableBuilder.cs @@ -10,6 +10,17 @@ namespace Implab.Automaton { void MarkFinalState(int state); void SetInitialState(int s); + + /// + /// Increases if needed the input alphabet size to hold the specified symbol. + /// + /// + /// + /// AlphabetSize = Math.Max(AlphabetSize, symbol + 1) + /// + /// + /// Symbol. + void AddSymbol(int symbol); } } diff --git a/Implab/Automaton/RegularExpressions/RegularDFA.cs b/Implab/Automaton/RegularExpressions/RegularDFA.cs --- a/Implab/Automaton/RegularExpressions/RegularDFA.cs +++ b/Implab/Automaton/RegularExpressions/RegularDFA.cs @@ -66,6 +66,9 @@ namespace Implab.Automaton.RegularExpres // skip all unclassified symbols foreach (var pair in alphaMap.Where(x => x.Value != 0)) alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value); + + var orig = ToString(); + var opt = dfa.ToString(); return dfa; } @@ -78,6 +81,15 @@ namespace Implab.Automaton.RegularExpres return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet(g)); } + public override string ToString() { + var states = new MapAlphabet(false, null); + + for (int i = 0; i < StateCount; i++) + states.DefineSymbol(string.Format("s{0}", i), i); + + return string.Format("//[RegularDFA {1} x {2}]\n{0}", PrintDFA(InputAlphabet, states),StateCount, AlphabetSize); + } + } } diff --git a/Implab/Components/LazyAndWeak.cs b/Implab/Components/LazyAndWeak.cs --- a/Implab/Components/LazyAndWeak.cs +++ b/Implab/Components/LazyAndWeak.cs @@ -7,7 +7,7 @@ namespace Implab.Components { /// /// /// Usefull when dealing with memory-intensive objects which are frequently used. - /// This class is similar to except is a singleton. + /// This class is similar to except it is a singleton. /// public class LazyAndWeak where T : class { @@ -44,6 +44,7 @@ namespace Implab.Components { } else { lock (m_lock) { // double check + weak = m_reference; if (weak != null) { value = weak.Target as T; if (value != null) diff --git a/Implab/Formats/JSON/JSONGrammar.cs b/Implab/Formats/JSON/JSONGrammar.cs --- a/Implab/Formats/JSON/JSONGrammar.cs +++ b/Implab/Formats/JSON/JSONGrammar.cs @@ -108,7 +108,7 @@ namespace Implab.Formats.JSON { } Token SymbolRangeToken(char start, char stop) { - return SymbolToken(Enumerable.Range(start,stop - start).Select(x => (char)x)); + return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x)); } protected override IndexedAlphabetBase CreateAlphabet() { diff --git a/Implab/Formats/StringScanner.cs b/Implab/Formats/StringScanner.cs --- a/Implab/Formats/StringScanner.cs +++ b/Implab/Formats/StringScanner.cs @@ -4,22 +4,14 @@ namespace Implab.Formats { public class StringScanner: TextScanner { const int CHUNK_SIZE = 1024; - readonly string m_text; - int m_pos; - - public StringScanner(string text) : base(text.Length, text.Length < CHUNK_SIZE ? text.Length : CHUNK_SIZE) { - m_text = text; - Feed(); + public StringScanner(string text) : base(null) { + Safe.ArgumentNotNull(text, "text"); + var data = text.ToCharArray(); + Feed(data, 0, data.Length); } protected override int Read(char[] buffer, int offset, int size) { - var actual = size + m_pos > m_text.Length ? m_text.Length - m_pos : size; - - m_text.CopyTo(m_pos,buffer,offset, actual); - - m_pos += actual; - - return actual; + return 0; } } } diff --git a/Implab/Formats/TextScanner.cs b/Implab/Formats/TextScanner.cs --- a/Implab/Formats/TextScanner.cs +++ b/Implab/Formats/TextScanner.cs @@ -53,29 +53,24 @@ namespace Implab.Formats { tag = null; var maxSymbol = alphabet.Length - 1; - + int next; do { // after the next chunk is read the offset in the buffer may change int pos = m_bufferOffset + m_tokenLength; - + next = state; while (pos < m_bufferSize) { var ch = m_buffer[pos]; - try { - var next = dfa[state, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]]; + next = dfa[next, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]]; if (next == AutomatonConst.UNREACHABLE_STATE) break; - + state = next; - }catch { - throw; - } pos++; } - m_tokenLength = pos - m_bufferOffset; - } while (state != AutomatonConst.UNREACHABLE_STATE && Feed()); + } while (next != AutomatonConst.UNREACHABLE_STATE && Feed()); m_tokenOffset = m_bufferOffset; m_bufferOffset += m_tokenLength; @@ -150,7 +145,7 @@ namespace Implab.Formats { } public void CopyTokenTo(char[] buffer, int offset) { - m_buffer.CopyTo(buffer, offset); + Array.Copy(m_buffer, m_tokenOffset,buffer, offset, m_tokenLength); } public void CopyTokenTo(StringBuilder sb) {