##// END OF EJS Templates
refactoring complete, JSONParser rewritten
cin -
r180:c32688129f14 ref20160224
parent child
Show More
@@ -15,3 +15,5 Implab.Diagnostics.Interactive/bin/
15 15 Implab.Diagnostics.Interactive/obj/
16 16 MonoPlay/bin/
17 17 MonoPlay/obj/
18 Implab.Test/Implab.Format.Test/bin/
19 Implab.Test/Implab.Format.Test/obj/
@@ -77,7 +77,7 namespace Implab.Automaton {
77 77 }
78 78
79 79 public bool Remove(AutomatonTransition item) {
80 m_transitions.Remove(item);
80 return m_transitions.Remove(item);
81 81 }
82 82
83 83 public int Count {
@@ -168,9 +168,9 namespace Implab.Automaton {
168 168
169 169 var rmap = m_transitions
170 170 .GroupBy(t => t.s2)
171 .ToLookup(
171 .ToDictionary(
172 172 g => g.Key, // s2
173 g => g.ToLookup(t => t.edge, t => t.s1)
173 g => g.GroupBy(t => t.edge, t => t.s1).ToDictionary(p => p.Key)
174 174 );
175 175
176 176 while (queue.Count > 0) {
@@ -180,7 +180,7 namespace Implab.Automaton {
180 180 for (int c = 0; c < m_symbolCount; c++) {
181 181 var stateX = new HashSet<int>();
182 182 foreach(var a in stateA)
183 stateX.UnionWith(rmap[a][c]); // all states from wich 'c' leads to 'a'
183 stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
184 184
185 185 foreach (var stateY in optimalStates.ToArray()) {
186 186 if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
@@ -62,9 +62,5 namespace Implab.Automaton {
62 62 return symbol.ToInt32(CultureInfo.InvariantCulture);
63 63 }
64 64
65 public override IEnumerable<T> InputSymbols {
66 get { return _symbols.Value; }
67 }
68
69 65 }
70 66 }
@@ -30,9 +30,9 namespace Implab.Automaton {
30 30 /// </remarks>
31 31 /// <returns>The translation map.</returns>
32 32 public int[] GetTranslationMap() {
33 Dictionary<int,int> map = new Dictionary<int, int>();
33 var map = new Dictionary<int, int>();
34 34
35 int max;
35 int max = 0;
36 36 foreach (var p in Mappings) {
37 37 var index = GetSymbolIndex(p.Key);
38 38 max = Math.Max(max, index);
@@ -53,7 +53,6 namespace Implab.Automaton.RegularExpres
53 53
54 54 var dfa = new RegularDFA<TInput, TTag>(alphabet);
55 55
56 var states = new DummyAlphabet(StateCount);
57 56 var alphaMap = new Dictionary<int,int>();
58 57 var stateMap = new Dictionary<int,int>();
59 58
@@ -2,6 +2,13
2 2 using System.Threading;
3 3
4 4 namespace Implab.Components {
5 /// <summary>
6 /// Creates an instace on-demand and allows it to be garbage collected.
7 /// </summary>
8 /// <remarks>
9 /// Usefull when dealing with memory-intensive objects which are frequently used.
10 /// This class is similar to <see cref="ObjectPool{T}"/> except is a singleton.
11 /// </remarks>
5 12 public class LazyAndWeak<T> where T : class {
6 13
7 14 readonly Func<T> m_factory;
@@ -35,6 +42,18 namespace Implab.Components {
35 42 if (Interlocked.CompareExchange(ref m_reference, new WeakReference(value), weak) == weak)
36 43 return value;
37 44 } else {
45 lock (m_lock) {
46 // double check
47 if (weak != null) {
48 value = weak.Target as T;
49 if (value != null)
50 return value;
51 }
52 // we are safe to write
53 value = m_factory();
54 m_reference = new WeakReference(value);
55 return value;
56 }
38 57 }
39 58 }
40 59 }
@@ -1,5 +1,5
1 1 using System;
2 using Implab.Parsing;
2 using Implab.Formats;
3 3
4 4 namespace Implab.Components {
5 5 public class RunnableComponent : Disposable, IRunnable, IInitializable {
@@ -5,6 +5,7
5 5 enum JSONElementContext {
6 6 None,
7 7 Object,
8 Array
8 Array,
9 Closed
9 10 }
10 11 }
@@ -2,6 +2,7
2 2 using Implab.Automaton.RegularExpressions;
3 3 using System;
4 4 using Implab.Automaton;
5 using Implab.Components;
5 6
6 7 namespace Implab.Formats.JSON {
7 8 class JSONGrammar : Grammar<char> {
@@ -23,7 +24,7 namespace Implab.Formats.JSON {
23 24 EscapedUnicode
24 25 }
25 26
26 static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();
27 static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
27 28
28 29 public static JSONGrammar Instance {
29 30 get { return _instance.Value; }
@@ -31,6 +32,7 namespace Implab.Formats.JSON {
31 32
32 33 readonly ScannerContext<TokenType> m_jsonExpression;
33 34 readonly ScannerContext<TokenType> m_stringExpression;
35 readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
34 36
35 37 public JSONGrammar() {
36 38 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
@@ -87,6 +89,12 namespace Implab.Formats.JSON {
87 89
88 90 }
89 91
92 protected override IAlphabetBuilder<char> AlphabetBuilder {
93 get {
94 return m_defaultAlphabet;
95 }
96 }
97
90 98 public ScannerContext<TokenType> JsonExpression {
91 99 get {
92 100 return m_jsonExpression;
@@ -103,7 +111,7 namespace Implab.Formats.JSON {
103 111 return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
104 112 }
105 113
106 protected override IAlphabetBuilder<char> CreateAlphabet() {
114 protected override IndexedAlphabetBase<char> CreateAlphabet() {
107 115 return new CharAlphabet();
108 116 }
109 117
@@ -5,17 +5,10 using Implab.Automaton;
5 5 using Implab.Automaton.RegularExpressions;
6 6 using System.Linq;
7 7 using Implab.Components;
8 using System.Collections.Generic;
8 9
9 10 namespace Implab.Formats.JSON {
10 11 /// <summary>
11 /// internal
12 /// </summary>
13 public struct JSONParserContext {
14 public string memberName;
15 public JSONElementContext elementContext;
16 }
17
18 /// <summary>
19 12 /// Pull парсер JSON данных.
20 13 /// </summary>
21 14 /// <remarks>
@@ -52,10 +45,11 namespace Implab.Formats.JSON {
52 45 }
53 46
54 47 public bool Move(JsonTokenType token) {
55 var next = m_dfa[m_state, token];
48 var next = m_dfa[m_state, (int)token];
56 49 if (next == AutomatonConst.UNREACHABLE_STATE)
57 50 return false;
58 51 m_state = next;
52 return true;
59 53 }
60 54
61 55 public JSONElementContext ElementContext {
@@ -63,40 +57,43 namespace Implab.Formats.JSON {
63 57 }
64 58 }
65 59
60 static readonly ParserContext _jsonContext;
61 static readonly ParserContext _objectContext;
62 static readonly ParserContext _arrayContext;
63
66 64 static JSONParser() {
67 65
68
69 var valueExpression = Token(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
70 var memberExpression = Token(JsonTokenType.String).Cat(Token(JsonTokenType.NameSeparator)).Cat(valueExpression);
66 var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
67 var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
71 68
72 69 var objectExpression = memberExpression
73 70 .Cat(
74 Token(JsonTokenType.ValueSeparator)
71 MakeToken(JsonTokenType.ValueSeparator)
75 72 .Cat(memberExpression)
76 73 .EClosure()
77 74 )
78 75 .Optional()
79 .Cat(Token(JsonTokenType.EndObject))
76 .Cat(MakeToken(JsonTokenType.EndObject))
80 77 .End();
81 78
82 79 var arrayExpression = valueExpression
83 80 .Cat(
84 Token(JsonTokenType.ValueSeparator)
81 MakeToken(JsonTokenType.ValueSeparator)
85 82 .Cat(valueExpression)
86 83 .EClosure()
87 84 )
88 85 .Optional()
89 .Cat(Token(JsonTokenType.EndArray))
86 .Cat(MakeToken(JsonTokenType.EndArray))
90 87 .End();
91 88
92 89 var jsonExpression = valueExpression.End();
93 90
94 _jsonDFA = CreateParserContext(jsonExpression, JSONElementContext.None);
95 _objectDFA = CreateParserContext(objectExpression, JSONElementContext.Object);
96 _arrayDFA = CreateParserContext(arrayExpression, JSONElementContext.Array);
91 _jsonContext = CreateParserContext(jsonExpression, JSONElementContext.None);
92 _objectContext = CreateParserContext(objectExpression, JSONElementContext.Object);
93 _arrayContext = CreateParserContext(arrayExpression, JSONElementContext.Array);
97 94 }
98 95
99 static Token Token(params JsonTokenType[] input) {
96 static Token MakeToken(params JsonTokenType[] input) {
100 97 return Token.New( input.Select(t => (int)t).ToArray() );
101 98 }
102 99
@@ -112,32 +109,36 namespace Implab.Formats.JSON {
112 109
113 110 #endregion
114 111
115 JSONScanner m_scanner;
112 readonly JSONScanner m_scanner;
116 113 MemberContext m_memberContext;
117 114
118 115 JSONElementType m_elementType;
119 116 object m_elementValue;
117 string m_memberName = String.Empty;
118
119 Stack<ParserContext> m_stack = new Stack<ParserContext>();
120 ParserContext m_context = _jsonContext;
120 121
121 122 /// <summary>
122 123 /// Создает новый парсер на основе строки, содержащей JSON
123 124 /// </summary>
124 125 /// <param name="text"></param>
125 public JSONParser(string text)
126 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
126 public JSONParser(string text) {
127 127 Safe.ArgumentNotEmpty(text, "text");
128 m_scanner = new JSONScanner();
129 m_scanner.Feed(text.ToCharArray());
128 m_scanner = new JSONScanner(text);
130 129 }
131 130
132 131 /// <summary>
133 132 /// Создает новый экземпляр парсера, на основе текстового потока.
134 133 /// </summary>
135 134 /// <param name="reader">Текстовый поток.</param>
136 public JSONParser(TextReader reader)
137 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
135 public JSONParser(TextReader reader) {
138 136 Safe.ArgumentNotNull(reader, "reader");
139 m_scanner = new JSONScanner();
140 m_scanner.Feed(reader, dispose);
137 m_scanner = new JSONScanner(reader);
138 }
139
140 public int Level {
141 get { return m_stack.Count; }
141 142 }
142 143
143 144 /// <summary>
@@ -152,7 +153,7 namespace Implab.Formats.JSON {
152 153 /// пустая строка.
153 154 /// </summary>
154 155 public string ElementName {
155 get { return m_context.info.memberName; }
156 get { return m_memberName; }
156 157 }
157 158
158 159 /// <summary>
@@ -167,55 +168,51 namespace Implab.Formats.JSON {
167 168 /// </summary>
168 169 /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
169 170 public bool Read() {
170 if (m_context.current == UNREACHEBLE_STATE)
171 throw new InvalidOperationException("The parser is in invalid state");
172 171 object tokenValue;
173 172 JsonTokenType tokenType;
174 m_context.info.memberName = String.Empty;
173
174 m_memberName = String.Empty;
175
175 176 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
176 Move((int)tokenType);
177 if (m_context.current == UNREACHEBLE_STATE)
177 if(!m_context.Move(tokenType))
178 178 UnexpectedToken(tokenValue, tokenType);
179
179 180 switch (tokenType) {
180 181 case JsonTokenType.BeginObject:
181 Switch(
182 _objectDFA,
183 INITIAL_STATE,
184 new JSONParserContext {
185 memberName = m_context.info.memberName,
186 elementContext = JSONElementContext.Object
187 }
188 );
182 m_stack.Push(m_context);
183 m_context = _objectContext;
184
189 185 m_elementValue = null;
190 186 m_memberContext = MemberContext.MemberName;
191 187 m_elementType = JSONElementType.BeginObject;
192 188 return true;
193 189 case JsonTokenType.EndObject:
194 Restore();
190 if (m_stack.Count == 0)
191 UnexpectedToken(tokenValue, tokenType);
192 m_context = m_stack.Pop();
193
195 194 m_elementValue = null;
196 195 m_elementType = JSONElementType.EndObject;
197 196 return true;
198 197 case JsonTokenType.BeginArray:
199 Switch(
200 _arrayDFA,
201 INITIAL_STATE,
202 new JSONParserContext {
203 memberName = m_context.info.memberName,
204 elementContext = JSONElementContext.Array
205 }
206 );
198 m_stack.Push(m_context);
199 m_context = _arrayContext;
200
207 201 m_elementValue = null;
208 202 m_memberContext = MemberContext.MemberValue;
209 203 m_elementType = JSONElementType.BeginArray;
210 204 return true;
211 205 case JsonTokenType.EndArray:
212 Restore();
206 if (m_stack.Count == 0)
207 UnexpectedToken(tokenValue, tokenType);
208 m_context = m_stack.Pop();
209
213 210 m_elementValue = null;
214 211 m_elementType = JSONElementType.EndArray;
215 212 return true;
216 213 case JsonTokenType.String:
217 214 if (m_memberContext == MemberContext.MemberName) {
218 m_context.info.memberName = (string)tokenValue;
215 m_memberName = (string)tokenValue;
219 216 break;
220 217 }
221 218 m_elementType = JSONElementType.Value;
@@ -233,15 +230,18 namespace Implab.Formats.JSON {
233 230 m_memberContext = MemberContext.MemberValue;
234 231 break;
235 232 case JsonTokenType.ValueSeparator:
236 m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
233 m_memberContext = m_context.ElementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
237 234 break;
238 235 default:
239 236 UnexpectedToken(tokenValue, tokenType);
240 237 break;
241 238 }
242 239 }
243 if (m_context.info.elementContext != JSONElementContext.None)
240 if (m_context.ElementContext != JSONElementContext.None)
244 241 throw new ParserException("Unexpedted end of data");
242
243 EOF = true;
244
245 245 return false;
246 246 }
247 247
@@ -268,15 +268,13 namespace Implab.Formats.JSON {
268 268 /// Признак конца потока
269 269 /// </summary>
270 270 public bool EOF {
271 get {
272 return m_scanner.EOF;
273 }
271 get;
272 private set;
274 273 }
275 274
276 275 protected override void Dispose(bool disposing) {
277 if (disposing) {
278 m_scanner.Dispose();
279 }
276 if (disposing)
277 Safe.Dispose(m_scanner);
280 278 }
281 279
282 280 /// <summary>
@@ -33,6 +33,9 namespace Implab.Formats.JSON {
33 33 m_scanner = new ReaderScanner(reader, bufferMax, chunkSize);
34 34 }
35 35
36 public JSONScanner(TextReader reader) : this(reader, 1024*1024, 1024){
37 }
38
36 39 /// <summary>
37 40 /// Читает следующий лексический элемент из входных данных.
38 41 /// </summary>
@@ -4,7 +4,7 using System.IO;
4 4 using System.Globalization;
5 5 using System.Diagnostics;
6 6
7 namespace Implab.JSON {
7 namespace Implab.Formats.JSON {
8 8 public class JSONWriter {
9 9 struct Context {
10 10 public bool needComma;
@@ -1,15 +1,11
1 1 using Implab;
2 using Implab.Parsing;
3 2 using System;
4 3 using System.Collections.Generic;
5 4 using System.Globalization;
6 5 using System.IO;
7 using System.Linq;
8 using System.Text;
9 using System.Threading.Tasks;
10 6 using System.Xml;
11 7
12 namespace Implab.JSON {
8 namespace Implab.Formats.JSON {
13 9 public class JSONXmlReader : XmlReader {
14 10
15 11 enum ValueContext {
@@ -30,7 +26,7 namespace Implab.JSON {
30 26 ReadState m_state = ReadState.Initial;
31 27 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
32 28 LocalNameContext m_localName;
33 int m_depthCorrection = 0;
29 int m_depthCorrection;
34 30
35 31 readonly string m_rootName;
36 32 readonly string m_prefix;
@@ -119,7 +115,7 namespace Implab.JSON {
119 115 public override string LookupNamespace(string prefix) {
120 116 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
121 117 return m_namespaceUri;
122 else
118
123 119 return String.Empty;
124 120 }
125 121
@@ -183,11 +179,11 namespace Implab.JSON {
183 179 }
184 180
185 181 public override bool Read() {
186 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
182 if (m_state != ReadState.Interactive && m_state != ReadState.Initial)
187 183 return false;
188 184
189 185 if (m_state == ReadState.Initial)
190 m_state = System.Xml.ReadState.Interactive;
186 m_state = ReadState.Interactive;
191 187
192 188 try {
193 189 switch (m_parser.ElementType) {
@@ -223,9 +219,8 namespace Implab.JSON {
223 219 m_depthCorrection--;
224 220 SetLocalName(itemName, true);
225 221 continue;
226 } else {
222 }
227 223 SetLocalName(itemName, true);
228 }
229 224 break;
230 225 case JSONElementType.BeginObject:
231 226 SetLocalName(itemName);
@@ -243,16 +238,14 namespace Implab.JSON {
243 238 SetLocalName(itemName);
244 239 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
245 240 break;
246 default:
247 break;
248 241 }
249 242 return true;
250 243 }
251 244
252 m_state = System.Xml.ReadState.EndOfFile;
245 m_state = ReadState.EndOfFile;
253 246 return false;
254 247 } catch {
255 m_state = System.Xml.ReadState.Error;
248 m_state = ReadState.Error;
256 249 throw;
257 250 }
258 251 }
@@ -275,7 +268,6 namespace Implab.JSON {
275 268 return String.Empty;
276 269 if (Convert.GetTypeCode(m_parser.ElementValue) == TypeCode.Double)
277 270 return ((double)m_parser.ElementValue).ToString(CultureInfo.InvariantCulture);
278 else
279 271 return m_parser.ElementValue.ToString();
280 272 }
281 273 }
@@ -323,7 +315,7 namespace Implab.JSON {
323 315 /// The reader will be disposed when the XmlReader is disposed.
324 316 /// </remarks>
325 317 public static JSONXmlReader Create(TextReader reader, JSONXmlReaderOptions options) {
326 return new JSONXmlReader(new JSONParser(reader, true), options);
318 return new JSONXmlReader(new JSONParser(reader), options);
327 319 }
328 320
329 321 /// <summary>
@@ -1,10 +1,7
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
1
5 2 using System.Xml;
6 3
7 namespace Implab.JSON {
4 namespace Implab.Formats.JSON {
8 5 /// <summary>
9 6 /// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
10 7 /// интерпретации <c>JSON</c> документа.
@@ -11,7 +11,7 namespace Implab.Formats.JSON {
11 11 /// <summary>
12 12 /// Класс для преобразования экранированной строки JSON
13 13 /// </summary>
14 public class StringTranslator : TextScanner<JSONGrammar.TokenType> {
14 static class StringTranslator {
15 15 static readonly char[] _escMap;
16 16 static readonly int[] _hexMap;
17 17
@@ -34,49 +34,6 namespace Implab.Formats.JSON {
34 34
35 35 }
36 36
37 public StringTranslator() {
38 }
39
40 public string Translate(string data) {
41 Safe.ArgumentNotNull(data, "data");
42 return Translate(data.ToCharArray());
43 }
44
45 public string Translate(char[] data) {
46 Safe.ArgumentNotNull(data, "data");
47 return Translate(data, data.Length);
48 }
49
50 public string Translate(char[] data, int length) {
51 Safe.ArgumentNotNull(data, "data");
52 Safe.ArgumentInRange(length, 0, data.Length, "length");
53
54 var translated = new char[length];
55
56 Feed(data,length);
57
58 int pos = 0;
59
60 while (ReadTokenInternal()) {
61 switch ((JSONGrammar.TokenType)Tags[0]) {
62 case JSONGrammar.TokenType.UnescapedChar:
63 Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
64 pos += m_tokenLen;
65 break;
66 case JSONGrammar.TokenType.EscapedChar:
67 translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
68 pos++;
69 break;
70 case JSONGrammar.TokenType.EscapedUnicode:
71 translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
72 pos++;
73 break;
74 }
75 }
76
77 return new String(translated, 0, pos);
78 }
79
80 37 internal static char TranslateEscapedChar(char symbol) {
81 38 return _escMap[symbol];
82 39 }
@@ -49,8 +49,8 namespace Implab.Formats {
49 49 /// <param name="alphabet"></param>
50 50 /// <param name = "tag"></param>
51 51 internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
52 Safe.ArgumentNotNull();
53 52 m_tokenLength = 0;
53 tag = null;
54 54
55 55 var maxSymbol = alphabet.Length - 1;
56 56
@@ -122,7 +122,13 namespace Implab.Formats {
122 122 m_bufferOffset = 0;
123 123 m_bufferSize = used + read;
124 124 m_buffer = temp;
125 } else {
126 var read = Read(m_buffer, m_bufferSize, m_chunkSize);
127 if (read == 0)
128 return false;
129 m_bufferSize += m_chunkSize;
125 130 }
131 return true;
126 132 } else {
127 133 Debug.Assert(m_bufferOffset == 0);
128 134 m_buffer = new char[m_chunkSize];
@@ -151,7 +151,6
151 151 <Compile Include="Components\ExecutionState.cs" />
152 152 <Compile Include="Components\RunnableComponent.cs" />
153 153 <Compile Include="Components\IFactory.cs" />
154 <Compile Include="Automaton\EnumAlphabet.cs" />
155 154 <Compile Include="Automaton\IAlphabet.cs" />
156 155 <Compile Include="Automaton\ParserException.cs" />
157 156 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
@@ -176,7 +175,6
176 175 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
177 176 <Compile Include="Formats\JSON\StringTranslator.cs" />
178 177 <Compile Include="Automaton\MapAlphabet.cs" />
179 <Compile Include="Automaton\DummyAlphabet.cs" />
180 178 <Compile Include="Formats\CharAlphabet.cs" />
181 179 <Compile Include="Formats\ByteAlphabet.cs" />
182 180 <Compile Include="Automaton\IDFATable.cs" />
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now