##// END OF EJS Templates
implab » ImplabNet
RSS
sync
cin -
r179:478ef706906a ref20160224
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,9 +1,9
1 1
2 2 namespace Implab.Automaton {
3 public static class DFAConst {
3 public static class AutomatonConst {
4 4 public const int UNREACHABLE_STATE = -1;
5 5
6 6 public const int UNCLASSIFIED_INPUT = 0;
7 7 }
8 8 }
9 9
Show file before | Show file after | Hide comments
@@ -1,83 +1,83
1 1 using System.Collections.Generic;
2 2 using System.Linq;
3 3
4 4 namespace Implab.Automaton.RegularExpressions {
5 public class TaggedDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
5 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
6 6
7 7 readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
8 8 readonly IAlphabet<TInput> m_alphabet;
9 9
10 public TaggedDFA(IAlphabet<TInput> alphabet) {
10 public RegularDFA(IAlphabet<TInput> alphabet) {
11 11 Safe.ArgumentNotNull(alphabet, "aplhabet");
12 12
13 13 m_alphabet = alphabet;
14 14 }
15 15
16 16
17 17 public IAlphabet<TInput> InputAlphabet {
18 18 get {
19 19 return m_alphabet;
20 20 }
21 21 }
22 22
23 23 public void MarkFinalState(int s, TTag[] tags) {
24 24 MarkFinalState(s);
25 25 SetStateTag(s, tags);
26 26 }
27 27
28 28 public void SetStateTag(int s, TTag[] tags) {
29 29 Safe.ArgumentNotNull(tags, "tags");
30 30 m_tags[s] = tags;
31 31 }
32 32
33 33 public TTag[] GetStateTag(int s) {
34 34 TTag[] tags;
35 35 return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
36 36 }
37 37
38 38 public TTag[][] CreateTagTable() {
39 39 var table = new TTag[StateCount][];
40 40
41 41 foreach (var pair in m_tags)
42 42 table[pair.Key] = pair.Value;
43 43
44 44 return table;
45 45 }
46 46
47 47 /// <summary>
48 48 /// Optimize the specified alphabet.
49 49 /// </summary>
50 50 /// <param name="alphabet">ΠŸΡƒΡΡ‚ΠΎΠΉ Π°Π»Ρ„Π°Π²ΠΈΡ‚, ΠΊΠΎΡ‚ΠΎΡ€Ρ‹ΠΉ Π±ΡƒΠ΄Π΅Ρ‚ Π·ΠΏΠΎΠ»Π½Π΅Π½ Π² процСссС ΠΎΠΏΡ‚ΠΈΠΌΠΈΠ·Π°Ρ†ΠΈΠΈ.</param>
51 public TaggedDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
51 public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
52 52 Safe.ArgumentNotNull(alphabet, "alphabet");
53 53
54 var dfa = new TaggedDFA<TInput, TTag>(alphabet);
54 var dfa = new RegularDFA<TInput, TTag>(alphabet);
55 55
56 56 var states = new DummyAlphabet(StateCount);
57 57 var alphaMap = new Dictionary<int,int>();
58 58 var stateMap = new Dictionary<int,int>();
59 59
60 60 Optimize(dfa, alphaMap, stateMap);
61 61
62 62 // mark tags in the new DFA
63 63 foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
64 64 dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
65 65
66 66 // make the alphabet for the new DFA
67 67 foreach (var pair in alphaMap)
68 68 alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
69 69
70 70 return dfa;
71 71 }
72 72
73 73 protected override IEnumerable<HashSet<int>> GroupFinalStates() {
74 74 var arrayComparer = new CustomEqualityComparer<TTag[]>(
75 75 (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
76 76 x => x.Sum(it => x.GetHashCode())
77 77 );
78 78 return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
79 79 }
80 80
81 81 }
82 82 }
83 83
Show file before | Show file after | Hide comments
@@ -1,104 +1,104
1 1 using System;
2 2 using System.Globalization;
3 3 using Implab.Automaton;
4 4 using System.Text;
5 5 using Implab.Components;
6 6 using System.IO;
7 7
8 8 namespace Implab.Formats.JSON {
9 9 /// <summary>
10 10 /// Π‘ΠΊΠ°Π½Π½Π΅Ρ€ (лСксСр), Ρ€Π°Π·Π±ΠΈΠ²Π°ΡŽΡ‰ΠΈΠΉ ΠΏΠΎΡ‚ΠΎΠΊ символов Π½Π° Ρ‚ΠΎΠΊΠ΅Π½Ρ‹ JSON.
11 11 /// </summary>
12 12 public class JSONScanner : Disposable {
13 13 readonly StringBuilder m_builder = new StringBuilder();
14 14
15 readonly ScannerContext<JSONGrammar.TokenType> m_jsonContext = JSONGrammar.Instance.JsonDFA;
16 readonly ScannerContext<JSONGrammar.TokenType> m_stringContext = JSONGrammar.Instance.JsonStringDFA;
15 readonly ScannerContext<JSONGrammar.TokenType> m_jsonContext = JSONGrammar.Instance.JsonExpression;
16 readonly ScannerContext<JSONGrammar.TokenType> m_stringContext = JSONGrammar.Instance.JsonStringExpression;
17 17
18 18
19 19 readonly TextScanner m_scanner;
20 20
21 21 /// <summary>
22 22 /// Π‘ΠΎΠ·Π΄Π°Π΅Ρ‚ Π½ΠΎΠ²Ρ‹ΠΉ экзСмпляр сканнСра
23 23 /// </summary>
24 24 public JSONScanner(string text) {
25 25 Safe.ArgumentNotEmpty(text, "text");
26 26
27 27 m_scanner = new StringScanner(text);
28 28 }
29 29
30 30 public JSONScanner(TextReader reader, int bufferMax, int chunkSize) {
31 31 Safe.ArgumentNotNull(reader, "reader");
32 32
33 33 m_scanner = new ReaderScanner(reader, bufferMax, chunkSize);
34 34 }
35 35
36 36 /// <summary>
37 37 /// Π§ΠΈΡ‚Π°Π΅Ρ‚ ΡΠ»Π΅Π΄ΡƒΡŽΡ‰ΠΈΠΉ лСксичСский элСмСнт ΠΈΠ· Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ….
38 38 /// </summary>
39 39 /// <param name="tokenValue">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Π·Π½Π°Ρ‡Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
40 40 /// <param name="tokenType">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Ρ‚ΠΈΠΏ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
41 41 /// <returns><c>true</c> - Ρ‡Ρ‚Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΠΈΠ·Π²Π΅Π΄Π΅Π½ΠΎ ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ. <c>false</c> - достигнут ΠΊΠΎΠ½Π΅Ρ† Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ…</returns>
42 42 /// <remarks>Π’ случС Ссли Ρ‚ΠΎΠΊΠ΅Π½ Π½Π΅ распознаСтся, Π²ΠΎΠ·Π½ΠΈΠΊΠ°Π΅Ρ‚ ΠΈΡΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅. ЗначСния Ρ‚ΠΎΠΊΠ΅Π½ΠΎΠ² ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ, Ρ‚.Π΅.
43 43 /// Π² строках ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ экранированныС символы, числа становтся Ρ‚ΠΈΠΏΠ° double.</remarks>
44 44 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
45 45 JSONGrammar.TokenType[] tag;
46 46 if (m_jsonContext.Execute(m_scanner, out tag)) {
47 47 switch (tag[0]) {
48 48 case JSONGrammar.TokenType.StringBound:
49 49 tokenValue = ReadString();
50 50 tokenType = JsonTokenType.String;
51 51 break;
52 52 case JSONGrammar.TokenType.Number:
53 53 tokenValue = Double.Parse(m_scanner.GetTokenValue(), CultureInfo.InvariantCulture);
54 54 tokenType = JsonTokenType.Number;
55 55 break;
56 56 default:
57 57 tokenType = (JsonTokenType)tag[0];
58 58 tokenValue = m_scanner.GetTokenValue();
59 59 break;
60 60 }
61 61 return true;
62 62 }
63 63 tokenValue = null;
64 64 tokenType = JsonTokenType.None;
65 65 return false;
66 66 }
67 67
68 68 string ReadString() {
69 69 int pos = 0;
70 70 var buf = new char[6]; // the buffer for unescaping chars
71 71
72 72 JSONGrammar.TokenType[] tag;
73 73 m_builder.Clear();
74 74
75 75 while (m_stringContext.Execute(m_scanner, out tag)) {
76 76 switch (tag[0]) {
77 77 case JSONGrammar.TokenType.StringBound:
78 78 return m_builder.ToString();
79 79 case JSONGrammar.TokenType.UnescapedChar:
80 80 m_scanner.CopyTokenTo(m_builder);
81 81 break;
82 82 case JSONGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
83 83 m_scanner.CopyTokenTo(buf, 0);
84 84 m_builder.Append(StringTranslator.TranslateHexUnicode(buf, 2));
85 85 pos++;
86 86 break;
87 87 case JSONGrammar.TokenType.EscapedChar: // \t - escape sequence
88 88 m_scanner.CopyTokenTo(buf, 0);
89 89 m_builder.Append(StringTranslator.TranslateEscapedChar(buf[1]));
90 90 break;
91 91 }
92 92
93 93 }
94 94
95 95 throw new ParserException("Unexpected end of data");
96 96 }
97 97
98 98 protected override void Dispose(bool disposing) {
99 99 if (disposing)
100 100 Safe.Dispose(m_scanner);
101 101 base.Dispose(disposing);
102 102 }
103 103 }
104 104 }
General Comments 0
You need to be logged in to leave comments. Login now