@@ -19,6 +19,7 | |||||
19 | <ErrorReport>prompt</ErrorReport> |
|
19 | <ErrorReport>prompt</ErrorReport> | |
20 | <WarningLevel>4</WarningLevel> |
|
20 | <WarningLevel>4</WarningLevel> | |
21 | <ConsolePause>false</ConsolePause> |
|
21 | <ConsolePause>false</ConsolePause> | |
|
22 | <RunCodeAnalysis>true</RunCodeAnalysis> | |||
22 | </PropertyGroup> |
|
23 | </PropertyGroup> | |
23 | <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> |
|
24 | <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> | |
24 | <DebugType>full</DebugType> |
|
25 | <DebugType>full</DebugType> |
@@ -3,6 +3,7 using Implab.Parsing; | |||||
3 | using System; |
|
3 | using System; | |
4 | using System.Collections.Generic; |
|
4 | using System.Collections.Generic; | |
5 | using System.Diagnostics; |
|
5 | using System.Diagnostics; | |
|
6 | using System.IO; | |||
6 | using System.Linq; |
|
7 | using System.Linq; | |
7 | using System.Text; |
|
8 | using System.Text; | |
8 | using System.Threading.Tasks; |
|
9 | using System.Threading.Tasks; | |
@@ -19,7 +20,7 namespace Implab.JSON { | |||||
19 | /// <summary> |
|
20 | /// <summary> | |
20 | /// Pull парсер JSON данных. |
|
21 | /// Pull парсер JSON данных. | |
21 | /// </summary> |
|
22 | /// </summary> | |
22 | public class JSONParser : DFAutomaton<JSONParserContext> { |
|
23 | public class JSONParser : DFAutomaton<JSONParserContext>, IDisposable { | |
23 |
|
24 | |||
24 | enum MemberContext { |
|
25 | enum MemberContext { | |
25 | MemberName, |
|
26 | MemberName, | |
@@ -75,6 +76,10 namespace Implab.JSON { | |||||
75 | JSONElementType m_elementType; |
|
76 | JSONElementType m_elementType; | |
76 | object m_elementValue; |
|
77 | object m_elementValue; | |
77 |
|
78 | |||
|
79 | /// <summary> | |||
|
80 | /// Создает новый парсер на основе строки, содержащей JSON | |||
|
81 | /// </summary> | |||
|
82 | /// <param name="text"></param> | |||
78 | public JSONParser(string text) |
|
83 | public JSONParser(string text) | |
79 |
: base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty } |
|
84 | : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) { | |
80 | Safe.ArgumentNotEmpty(text, "text"); |
|
85 | Safe.ArgumentNotEmpty(text, "text"); | |
@@ -82,18 +87,44 namespace Implab.JSON { | |||||
82 | m_scanner.Feed(text.ToCharArray()); |
|
87 | m_scanner.Feed(text.ToCharArray()); | |
83 | } |
|
88 | } | |
84 |
|
89 | |||
|
90 | /// <summary> | |||
|
91 | /// Создает новый экземпляр парсера, на основе текстового потока. | |||
|
92 | /// </summary> | |||
|
93 | /// <param name="reader">Текстовый поток.</param> | |||
|
94 | /// <param name="dispose">Признак того, что парсер должен конролировать время жизни входного потока.</param> | |||
|
95 | public JSONParser(TextReader reader, bool dispose) | |||
|
96 | : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) { | |||
|
97 | Safe.ArgumentNotNull(reader, "reader"); | |||
|
98 | m_scanner = new JSONScanner(); | |||
|
99 | m_scanner.Feed(reader, dispose); | |||
|
100 | } | |||
|
101 | ||||
|
102 | /// <summary> | |||
|
103 | /// Тип текущего элемента на котором стоит парсер. | |||
|
104 | /// </summary> | |||
85 | public JSONElementType ElementType { |
|
105 | public JSONElementType ElementType { | |
86 | get { return m_elementType; } |
|
106 | get { return m_elementType; } | |
87 | } |
|
107 | } | |
88 |
|
108 | |||
|
109 | /// <summary> | |||
|
110 | /// Имя элемента - имя свойства родительского контейнера. Для элементов массивов и корневого всегда | |||
|
111 | /// пустая строка. | |||
|
112 | /// </summary> | |||
89 | public string ElementName { |
|
113 | public string ElementName { | |
90 | get { return m_context.info.memberName; } |
|
114 | get { return m_context.info.memberName; } | |
91 | } |
|
115 | } | |
92 |
|
116 | |||
|
117 | /// <summary> | |||
|
118 | /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c> | |||
|
119 | /// </summary> | |||
93 | public object ElementValue { |
|
120 | public object ElementValue { | |
94 | get { return m_elementValue; } |
|
121 | get { return m_elementValue; } | |
95 | } |
|
122 | } | |
96 |
|
123 | |||
|
124 | /// <summary> | |||
|
125 | /// Читает слеюудущий объект из потока | |||
|
126 | /// </summary> | |||
|
127 | /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns> | |||
97 | public bool Read() { |
|
128 | public bool Read() { | |
98 | if (m_context.current == UNREACHEBLE_STATE) |
|
129 | if (m_context.current == UNREACHEBLE_STATE) | |
99 | throw new InvalidOperationException("The parser is in invalid state"); |
|
130 | throw new InvalidOperationException("The parser is in invalid state"); | |
@@ -193,11 +224,32 namespace Implab.JSON { | |||||
193 | } |
|
224 | } | |
194 |
|
225 | |||
195 |
|
226 | |||
|
227 | /// <summary> | |||
|
228 | /// Признак конца потока | |||
|
229 | /// </summary> | |||
196 | public bool EOF { |
|
230 | public bool EOF { | |
197 | get { |
|
231 | get { | |
198 | return m_scanner.EOF; |
|
232 | return m_scanner.EOF; | |
199 | } |
|
233 | } | |
200 | } |
|
234 | } | |
|
235 | ||||
|
236 | protected virtual void Dispose(bool disposing) { | |||
|
237 | if (disposing) { | |||
|
238 | m_scanner.Dispose(); | |||
|
239 | } | |||
|
240 | } | |||
|
241 | ||||
|
242 | /// <summary> | |||
|
243 | /// Освобождает парсер и связанный с ним сканнер. | |||
|
244 | /// </summary> | |||
|
245 | public void Dispose() { | |||
|
246 | Dispose(true); | |||
|
247 | GC.SuppressFinalize(this); | |||
|
248 | } | |||
|
249 | ||||
|
250 | ~JSONParser() { | |||
|
251 | Dispose(false); | |||
|
252 | } | |||
201 | } |
|
253 | } | |
202 |
|
254 | |||
203 | } |
|
255 | } |
@@ -272,7 +272,15 namespace ConsPlay { | |||||
272 | } |
|
272 | } | |
273 |
|
273 | |||
274 | public override void Close() { |
|
274 | public override void Close() { | |
275 | m_state = System.Xml.ReadState.EndOfFile ; |
|
275 | ||
|
276 | } | |||
|
277 | ||||
|
278 | protected override void Dispose(bool disposing) { | |||
|
279 | if (disposing) { | |||
|
280 | m_parser.Dispose(); | |||
|
281 | } | |||
|
282 | base.Dispose(disposing); | |||
|
283 | } | |||
|
284 | ||||
276 |
|
|
285 | } | |
277 | } |
|
286 | } | |
278 | } |
|
@@ -8,6 +8,10 using System.Threading.Tasks; | |||||
8 | namespace Implab.Parsing { |
|
8 | namespace Implab.Parsing { | |
9 | public class Alphabet: AlphabetBase<char> { |
|
9 | public class Alphabet: AlphabetBase<char> { | |
10 |
|
10 | |||
|
11 | public Alphabet() | |||
|
12 | : base(char.MaxValue + 1) { | |||
|
13 | } | |||
|
14 | ||||
11 | public override int GetSymbolIndex(char symbol) { |
|
15 | public override int GetSymbolIndex(char symbol) { | |
12 | return symbol; |
|
16 | return symbol; | |
13 | } |
|
17 | } | |
@@ -15,9 +19,5 namespace Implab.Parsing { | |||||
15 | public override IEnumerable<char> InputSymbols { |
|
19 | public override IEnumerable<char> InputSymbols { | |
16 | get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); } |
|
20 | get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); } | |
17 | } |
|
21 | } | |
18 |
|
||||
19 | protected override int MapSize { |
|
|||
20 | get { return char.MaxValue + 1; } |
|
|||
21 |
|
|
22 | } | |
22 | } |
|
23 | } | |
23 | } |
|
@@ -17,13 +17,12 namespace Implab.Parsing { | |||||
17 | get { return m_nextId; } |
|
17 | get { return m_nextId; } | |
18 | } |
|
18 | } | |
19 |
|
19 | |||
20 | protected AlphabetBase() { |
|
20 | protected AlphabetBase(int mapSize) { | |
21 |
m_map = new int[ |
|
21 | m_map = new int[mapSize]; | |
22 | } |
|
22 | } | |
23 |
|
23 | |||
24 | protected AlphabetBase(int[] map) { |
|
24 | protected AlphabetBase(int[] map) { | |
25 | Debug.Assert(map != null); |
|
25 | Debug.Assert(map != null); | |
26 | Debug.Assert(map.Length == MapSize); |
|
|||
27 |
|
26 | |||
28 | m_map = map; |
|
27 | m_map = map; | |
29 | m_nextId = map.Max() + 1; |
|
28 | m_nextId = map.Max() + 1; | |
@@ -94,8 +93,6 namespace Implab.Parsing { | |||||
94 |
|
93 | |||
95 | public abstract IEnumerable<T> InputSymbols { get; } |
|
94 | public abstract IEnumerable<T> InputSymbols { get; } | |
96 |
|
95 | |||
97 | protected abstract int MapSize { get; } |
|
|||
98 |
|
||||
99 | public int[] GetTranslationMap() { |
|
96 | public int[] GetTranslationMap() { | |
100 | return m_map; |
|
97 | return m_map; | |
101 | } |
|
98 | } |
@@ -1,6 +1,7 | |||||
1 | using Implab; |
|
1 | using Implab; | |
2 | using System; |
|
2 | using System; | |
3 | using System.Collections.Generic; |
|
3 | using System.Collections.Generic; | |
|
4 | using System.Diagnostics; | |||
4 | using System.Globalization; |
|
5 | using System.Globalization; | |
5 | using System.Linq; |
|
6 | using System.Linq; | |
6 | using System.Text; |
|
7 | using System.Text; | |
@@ -45,11 +46,12 namespace Implab.Parsing { | |||||
45 |
|
46 | |||
46 |
|
47 | |||
47 | public EnumAlphabet() |
|
48 | public EnumAlphabet() | |
48 | : base() { |
|
49 | : base(_symbols.Length) { | |
49 | } |
|
50 | } | |
50 |
|
51 | |||
51 | public EnumAlphabet(int[] map) |
|
52 | public EnumAlphabet(int[] map) | |
52 | : base(map) { |
|
53 | : base(map) { | |
|
54 | Debug.Assert(map.Length == _symbols.Length); | |||
53 | } |
|
55 | } | |
54 |
|
56 | |||
55 |
|
57 | |||
@@ -61,8 +63,5 namespace Implab.Parsing { | |||||
61 | get { return _symbols; } |
|
63 | get { return _symbols; } | |
62 | } |
|
64 | } | |
63 |
|
65 | |||
64 | protected override int MapSize { |
|
|||
65 | get { return _symbols.Length; } |
|
|||
66 |
|
|
66 | } | |
67 | } |
|
67 | } | |
68 | } |
|
@@ -1,6 +1,7 | |||||
1 | using Implab; |
|
1 | using Implab; | |
2 | using System; |
|
2 | using System; | |
3 | using System.Collections.Generic; |
|
3 | using System.Collections.Generic; | |
|
4 | using System.IO; | |||
4 | using System.Linq; |
|
5 | using System.Linq; | |
5 | using System.Text; |
|
6 | using System.Text; | |
6 | using System.Threading.Tasks; |
|
7 | using System.Threading.Tasks; | |
@@ -14,7 +15,7 namespace Implab.Parsing { | |||||
14 | /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения |
|
15 | /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения | |
15 | /// конца токена и допустимости текущего символа. |
|
16 | /// конца токена и допустимости текущего символа. | |
16 | /// </remarks> |
|
17 | /// </remarks> | |
17 | public class Scanner { |
|
18 | public abstract class Scanner : Disposable { | |
18 | struct ScannerConfig { |
|
19 | struct ScannerConfig { | |
19 | public DFAStateDescriptior[] states; |
|
20 | public DFAStateDescriptior[] states; | |
20 | public int[] alphabetMap; |
|
21 | public int[] alphabetMap; | |
@@ -35,15 +36,10 namespace Implab.Parsing { | |||||
35 | protected int m_bufferSize; |
|
36 | protected int m_bufferSize; | |
36 | protected int m_pointer; |
|
37 | protected int m_pointer; | |
37 |
|
38 | |||
38 | public Scanner(CDFADefinition definition, string text) { |
|
39 | TextReader m_reader; | |
39 | Safe.ArgumentNotNull(definition, "definition"); |
|
40 | bool m_disposeReader; | |
40 | Safe.ArgumentNotEmpty(text, "text"); |
|
41 | int m_chunkSize = 1024; // 1k | |
41 |
|
42 | int m_limit = 10 * 1024 * 1024; // 10Mb | ||
42 | m_states = definition.States; |
|
|||
43 | m_alphabetMap = definition.Alphabet.GetTranslationMap(); |
|
|||
44 |
|
||||
45 | Feed(text.ToCharArray()); |
|
|||
46 | } |
|
|||
47 |
|
43 | |||
48 | public Scanner(CDFADefinition definition) { |
|
44 | public Scanner(CDFADefinition definition) { | |
49 | Safe.ArgumentNotNull(definition, "definition"); |
|
45 | Safe.ArgumentNotNull(definition, "definition"); | |
@@ -76,6 +72,7 namespace Implab.Parsing { | |||||
76 | public void Feed(char[] data, int length) { |
|
72 | public void Feed(char[] data, int length) { | |
77 | Safe.ArgumentNotNull(data, "data"); |
|
73 | Safe.ArgumentNotNull(data, "data"); | |
78 | Safe.ArgumentInRange(length, 0, data.Length, "length"); |
|
74 | Safe.ArgumentInRange(length, 0, data.Length, "length"); | |
|
75 | AssertNotDisposed(); | |||
79 |
|
76 | |||
80 | m_pointer = -1; |
|
77 | m_pointer = -1; | |
81 | m_buffer = data; |
|
78 | m_buffer = data; | |
@@ -83,18 +80,33 namespace Implab.Parsing { | |||||
83 | Shift(); |
|
80 | Shift(); | |
84 | } |
|
81 | } | |
85 |
|
82 | |||
|
83 | public void Feed(TextReader reader, bool dispose) { | |||
|
84 | Safe.ArgumentNotNull(reader, "reader"); | |||
|
85 | AssertNotDisposed(); | |||
|
86 | ||||
|
87 | if (m_reader != null && m_disposeReader) | |||
|
88 | m_reader.Dispose(); | |||
|
89 | ||||
|
90 | m_reader = reader; | |||
|
91 | m_disposeReader = dispose; | |||
|
92 | m_pointer = -1; | |||
|
93 | m_buffer = new char[m_chunkSize]; | |||
|
94 | m_bufferSize = 0; | |||
|
95 | Shift(); | |||
|
96 | } | |||
|
97 | ||||
86 | /// <summary> |
|
98 | /// <summary> | |
87 | /// Получает текущий токен в виде строки. |
|
99 | /// Получает текущий токен в виде строки. | |
88 | /// </summary> |
|
100 | /// </summary> | |
89 | /// <returns></returns> |
|
101 | /// <returns></returns> | |
90 |
p |
|
102 | protected string GetTokenValue() { | |
91 | return new String(m_buffer, m_tokenOffset, m_tokenLen); |
|
103 | return new String(m_buffer, m_tokenOffset, m_tokenLen); | |
92 | } |
|
104 | } | |
93 |
|
105 | |||
94 | /// <summary> |
|
106 | /// <summary> | |
95 | /// Метки текущего токена, которые были назначены в регулярном выражении. |
|
107 | /// Метки текущего токена, которые были назначены в регулярном выражении. | |
96 | /// </summary> |
|
108 | /// </summary> | |
97 |
p |
|
109 | protected int[] TokenTags { | |
98 | get { |
|
110 | get { | |
99 | return m_currentState.tag; |
|
111 | return m_currentState.tag; | |
100 | } |
|
112 | } | |
@@ -163,13 +175,31 namespace Implab.Parsing { | |||||
163 | return true; |
|
175 | return true; | |
164 | } |
|
176 | } | |
165 |
|
177 | |||
166 | /// <summary> |
|
178 | bool ReadNextChunk() { | |
167 | /// Вызывается по достижению конца входного буффера для получения |
|
179 | if (m_reader == null) | |
168 | /// новых данных. |
|
|||
169 | /// </summary> |
|
|||
170 | /// <returns><c>true</c> - новые двнные получены, можно продолжать обработку.</returns> |
|
|||
171 | protected virtual bool ReadNextChunk() { |
|
|||
172 | return false; |
|
180 | return false; | |
|
181 | ||||
|
182 | // extend buffer if nesessary | |||
|
183 | if (m_pointer + m_chunkSize > m_buffer.Length) { | |||
|
184 | // trim unused buffer head | |||
|
185 | var size = m_tokenLen + m_chunkSize; | |||
|
186 | if (size >= m_limit) | |||
|
187 | throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit)); | |||
|
188 | var temp = new char[size]; | |||
|
189 | Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen); | |||
|
190 | m_pointer -= m_tokenOffset; | |||
|
191 | m_bufferSize -= m_tokenOffset; | |||
|
192 | m_tokenOffset = 0; | |||
|
193 | m_buffer = temp; | |||
|
194 | } | |||
|
195 | ||||
|
196 | var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize); | |||
|
197 | if (read == 0) | |||
|
198 | return false; | |||
|
199 | ||||
|
200 | m_bufferSize += read; | |||
|
201 | ||||
|
202 | return true; | |||
173 | } |
|
203 | } | |
174 |
|
204 | |||
175 | /// <summary> |
|
205 | /// <summary> | |
@@ -212,5 +242,18 namespace Implab.Parsing { | |||||
212 | m_alphabetMap = prev.alphabetMap; |
|
242 | m_alphabetMap = prev.alphabetMap; | |
213 | m_previewCode = m_alphabetMap[m_buffer[m_pointer]]; |
|
243 | m_previewCode = m_alphabetMap[m_buffer[m_pointer]]; | |
214 | } |
|
244 | } | |
|
245 | ||||
|
246 | protected override void Dispose(bool disposing) { | |||
|
247 | if (disposing) { | |||
|
248 | if (m_reader != null && m_disposeReader) | |||
|
249 | m_reader.Dispose(); | |||
|
250 | m_buffer = null; | |||
|
251 | m_bufferSize = 0; | |||
|
252 | m_pointer = 0; | |||
|
253 | m_tokenLen = 0; | |||
|
254 | m_tokenOffset = 0; | |||
|
255 | } | |||
|
256 | base.Dispose(disposing); | |||
215 | } |
|
257 | } | |
216 | } |
|
258 | } | |
|
259 | } |
@@ -1,5 +1,6 | |||||
1 | using System.Reflection; |
|
1 | using System.Reflection; | |
2 | using System.Runtime.CompilerServices; |
|
2 | using System.Runtime.CompilerServices; | |
|
3 | using System.Runtime.InteropServices; | |||
3 |
|
4 | |||
4 | // Information about this assembly is defined by the following attributes. |
|
5 | // Information about this assembly is defined by the following attributes. | |
5 | // Change them to the values specific to your project. |
|
6 | // Change them to the values specific to your project. | |
@@ -16,6 +17,7 using System.Runtime.CompilerServices; | |||||
16 | // and "{Major}.{Minor}.{Build}.*" will update just the revision. |
|
17 | // and "{Major}.{Minor}.{Build}.*" will update just the revision. | |
17 |
|
18 | |||
18 | [assembly: AssemblyVersion("1.0.*")] |
|
19 | [assembly: AssemblyVersion("1.0.*")] | |
|
20 | [assembly: ComVisible(false)] | |||
19 |
|
21 | |||
20 | // The following attributes are used to specify the signing key for the assembly, |
|
22 | // The following attributes are used to specify the signing key for the assembly, | |
21 | // if desired. See the Mono documentation for more information about signing. |
|
23 | // if desired. See the Mono documentation for more information about signing. |
General Comments 0
You need to be logged in to leave comments.
Login now