##// END OF EJS Templates
implab » ImplabNet
RSS
+JSONXmlReaderOptions...
cin -
r60:10c7337d29e7 default
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,61
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Xml;
6
7 namespace Implab.JSON {
8 /// <summary>
9 /// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
10 /// интерпретации <c>JSON</c> документа.
11 /// </summary>
12 public class JSONXmlReaderOptions {
13 /// <summary>
14 /// Пространство имен в котором будут располагаться читаемые элементы документа
15 /// </summary>
16 public string NamespaceURI {
17 get;
18 set;
19 }
20
21 /// <summary>
22 /// Интерпретировать массивы как множественные элементы (убирает один уровень вложенности)
23 /// </summary>
24 public bool FlattenArrays {
25 get;
26 set;
27 }
28
29 /// <summary>
30 /// Префикс, для узлов документа
31 /// </summary>
32 public string NodesPrefix {
33 get;
34 set;
35 }
36
37 /// <summary>
38 /// Имя корневого элемента в xml документе
39 /// </summary>
40 public string RootName {
41 get;
42 set;
43 }
44
45 /// <summary>
46 /// Имя элемента для массивов, если не включена опция <see cref="FlattenArrays"/>.
47 /// </summary>
48 public string ArrayItemName {
49 get;
50 set;
51 }
52
53 /// <summary>
54 /// Таблица атомизированных строк для построения документа.
55 /// </summary>
56 public XmlNameTable NameTable {
57 get;
58 set;
59 }
60 }
61 }
Show file before | Show file after | Hide comments
@@ -1,105 +1,106
1 1 <?xml version="1.0" encoding="utf-8"?>
2 2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 3 <PropertyGroup>
4 4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5 5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6 6 <ProductVersion>10.0.0</ProductVersion>
7 7 <SchemaVersion>2.0</SchemaVersion>
8 8 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
9 9 <OutputType>Library</OutputType>
10 10 <RootNamespace>Implab</RootNamespace>
11 11 <AssemblyName>Implab</AssemblyName>
12 12 </PropertyGroup>
13 13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
14 14 <DebugSymbols>true</DebugSymbols>
15 15 <DebugType>full</DebugType>
16 16 <Optimize>false</Optimize>
17 17 <OutputPath>bin\Debug</OutputPath>
18 18 <DefineConstants>TRACE;DEBUG;</DefineConstants>
19 19 <ErrorReport>prompt</ErrorReport>
20 20 <WarningLevel>4</WarningLevel>
21 21 <ConsolePause>false</ConsolePause>
22 22 <RunCodeAnalysis>true</RunCodeAnalysis>
23 23 </PropertyGroup>
24 24 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
25 25 <DebugType>full</DebugType>
26 26 <Optimize>true</Optimize>
27 27 <OutputPath>bin\Release</OutputPath>
28 28 <ErrorReport>prompt</ErrorReport>
29 29 <WarningLevel>4</WarningLevel>
30 30 <ConsolePause>false</ConsolePause>
31 31 </PropertyGroup>
32 32 <ItemGroup>
33 33 <Reference Include="System" />
34 34 <Reference Include="System.XML" />
35 35 </ItemGroup>
36 36 <ItemGroup>
37 37 <Compile Include="Component.cs" />
38 38 <Compile Include="CustomEqualityComparer.cs" />
39 39 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
40 40 <Compile Include="Diagnostics\EventText.cs" />
41 41 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
42 42 <Compile Include="Diagnostics\LogChannel.cs" />
43 43 <Compile Include="Diagnostics\LogicalOperation.cs" />
44 44 <Compile Include="Diagnostics\TextFileListener.cs" />
45 45 <Compile Include="Diagnostics\TextListenerBase.cs" />
46 46 <Compile Include="Diagnostics\TraceLog.cs" />
47 47 <Compile Include="Diagnostics\TraceContext.cs" />
48 48 <Compile Include="Diagnostics\TraceEvent.cs" />
49 49 <Compile Include="Diagnostics\TraceEventType.cs" />
50 50 <Compile Include="Disposable.cs" />
51 51 <Compile Include="ICancellable.cs" />
52 52 <Compile Include="IProgressHandler.cs" />
53 53 <Compile Include="IProgressNotifier.cs" />
54 54 <Compile Include="IPromise.cs" />
55 55 <Compile Include="IPromiseBase.cs" />
56 56 <Compile Include="IServiceLocator.cs" />
57 57 <Compile Include="ITaskController.cs" />
58 58 <Compile Include="JSON\JSONElementContext.cs" />
59 59 <Compile Include="JSON\JSONElementType.cs" />
60 60 <Compile Include="JSON\JSONGrammar.cs" />
61 61 <Compile Include="JSON\JSONParser.cs" />
62 62 <Compile Include="JSON\JSONScanner.cs" />
63 63 <Compile Include="JSON\JsonTokenType.cs" />
64 64 <Compile Include="JSON\JSONWriter.cs" />
65 65 <Compile Include="JSON\JSONXmlReader.cs" />
66 <Compile Include="JSON\JSONXmlReaderOptions.cs" />
66 67 <Compile Include="JSON\StringTranslator.cs" />
67 68 <Compile Include="Parallels\DispatchPool.cs" />
68 69 <Compile Include="Parallels\ArrayTraits.cs" />
69 70 <Compile Include="Parallels\MTQueue.cs" />
70 71 <Compile Include="Parallels\WorkerPool.cs" />
71 72 <Compile Include="Parsing\Alphabet.cs" />
72 73 <Compile Include="Parsing\AlphabetBase.cs" />
73 74 <Compile Include="Parsing\AltToken.cs" />
74 75 <Compile Include="Parsing\BinaryToken.cs" />
75 76 <Compile Include="Parsing\CatToken.cs" />
76 77 <Compile Include="Parsing\CDFADefinition.cs" />
77 78 <Compile Include="Parsing\DFABuilder.cs" />
78 79 <Compile Include="Parsing\DFADefinitionBase.cs" />
79 80 <Compile Include="Parsing\DFAStateDescriptor.cs" />
80 81 <Compile Include="Parsing\DFAutomaton.cs" />
81 82 <Compile Include="Parsing\EDFADefinition.cs" />
82 83 <Compile Include="Parsing\EmptyToken.cs" />
83 84 <Compile Include="Parsing\EndToken.cs" />
84 85 <Compile Include="Parsing\EnumAlphabet.cs" />
85 86 <Compile Include="Parsing\Grammar.cs" />
86 87 <Compile Include="Parsing\IAlphabet.cs" />
87 88 <Compile Include="Parsing\IDFADefinition.cs" />
88 89 <Compile Include="Parsing\IVisitor.cs" />
89 90 <Compile Include="Parsing\ParserException.cs" />
90 91 <Compile Include="Parsing\Scanner.cs" />
91 92 <Compile Include="Parsing\StarToken.cs" />
92 93 <Compile Include="Parsing\SymbolToken.cs" />
93 94 <Compile Include="Parsing\Token.cs" />
94 95 <Compile Include="ServiceLocator.cs" />
95 96 <Compile Include="TaskController.cs" />
96 97 <Compile Include="ProgressInitEventArgs.cs" />
97 98 <Compile Include="Properties\AssemblyInfo.cs" />
98 99 <Compile Include="Promise.cs" />
99 100 <Compile Include="Parallels\AsyncPool.cs" />
100 101 <Compile Include="Safe.cs" />
101 102 <Compile Include="ValueEventArgs.cs" />
102 103 </ItemGroup>
103 104 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
104 105 <ItemGroup />
105 106 </Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,89 +1,100
1 1 using Implab.Parsing;
2 2 using System;
3 3 using System.Collections.Generic;
4 4 using System.Globalization;
5 5 using System.Linq;
6 6 using System.Text;
7 7 using System.Threading.Tasks;
8 8
9 9 namespace Implab.JSON {
10 10 /// <summary>
11 /// Сканнер, разбивающий поток символов на токены JSON.
11 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
12 12 /// </summary>
13 13 public class JSONScanner : Scanner {
14 14 char[] m_stringBuffer;
15 15 DFAStateDescriptior[] m_stringDFA;
16 16 int[] m_stringAlphabet;
17 17
18 /// <summary>
19 /// Создает новый экземпляр сканнера
20 /// </summary>
18 21 public JSONScanner()
19 22 : base(JSONGrammar.Instance.JsonDFA) {
20 23 m_stringBuffer = new char[1024];
21 24 var dfa = JSONGrammar.Instance.JsonStringDFA;
22 25 m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
23 26 m_stringDFA = dfa.States;
24 27 }
25 28
29 /// <summary>
30 /// Читает следующий лексический элемент из входных данных.
31 /// </summary>
32 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
33 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
34 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
35 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
36 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
26 37 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
27 38 if (ReadTokenInternal()) {
28 39 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
29 40 case JSONGrammar.TokenType.StringBound:
30 41 tokenValue = ReadString();
31 42 tokenType = JsonTokenType.String;
32 43 break;
33 44 case JSONGrammar.TokenType.Number:
34 45 tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
35 46 tokenType = JsonTokenType.Number;
36 47 break;
37 48 default:
38 49 tokenType = (JsonTokenType)m_currentState.tag[0];
39 50 tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
40 51 break;
41 52 }
42 53 return true;
43 54 }
44 55 tokenValue = null;
45 56 tokenType = JsonTokenType.None;
46 57 return false;
47 58 }
48 59
49 60 string ReadString() {
50 61 int pos = 0;
51 62 Switch(m_stringDFA, m_stringAlphabet);
52 63 while (ReadTokenInternal()) {
53 64 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
54 65 case JSONGrammar.TokenType.StringBound:
55 66 Restore();
56 67 return new String(m_stringBuffer, 0, pos);
57 68 case JSONGrammar.TokenType.UnescapedChar:
58 69 EnsureStringBufferSize(pos + m_tokenLen);
59 70 Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
60 71 pos += m_tokenLen;
61 72 break;
62 73 case JSONGrammar.TokenType.EscapedUnicode:
63 74 EnsureStringBufferSize(pos + 1);
64 75 m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
65 76 pos++;
66 77 break;
67 78 case JSONGrammar.TokenType.EscapedChar:
68 79 EnsureStringBufferSize(pos + 1);
69 80 m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
70 81 pos++;
71 82 break;
72 83 default:
73 84 break;
74 85 }
75 86
76 87 }
77 88
78 89 throw new ParserException("Unexpected end of data");
79 90 }
80 91
81 92 void EnsureStringBufferSize(int size) {
82 93 if (size > m_stringBuffer.Length) {
83 94 var newBuffer = new char[size];
84 95 m_stringBuffer.CopyTo(newBuffer, 0);
85 96 m_stringBuffer = newBuffer;
86 97 }
87 98 }
88 99 }
89 100 }
Show file before | Show file after | Hide comments
@@ -1,286 +1,310
1 1 using Implab;
2 using Implab.JSON;
3 2 using Implab.Parsing;
4 3 using System;
5 4 using System.Collections.Generic;
5 using System.IO;
6 6 using System.Linq;
7 7 using System.Text;
8 8 using System.Threading.Tasks;
9 9 using System.Xml;
10 10
11 namespace ConsPlay {
11 namespace Implab.JSON {
12 12 public class JSONXmlReader : XmlReader {
13 13
14 14 enum ValueContext {
15 15 Undefined,
16 16 ElementStart,
17 17 ElementValue,
18 18 ElementEnd,
19 19 ElementEmpty
20 20 }
21 21
22 22 struct LocalNameContext {
23 23 public string localName;
24 24 public bool isArray;
25 25 }
26 26
27 27 JSONParser m_parser;
28 28 ValueContext m_valueContext;
29 29 ReadState m_state = ReadState.Initial;
30 30 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
31 31 LocalNameContext m_localName;
32 string m_rootName = "json";
33 string m_prefix = String.Empty;
34 string m_namespaceUri = String.Empty;
35 bool m_flattenArrays = false;
36 NameTable m_nameTable = new NameTable();
37 32 int m_depthCorrection = 0;
38 33
39 public JSONXmlReader(JSONParser parser) {
34 readonly string m_rootName;
35 readonly string m_prefix;
36 readonly string m_namespaceUri;
37 readonly bool m_flattenArrays;
38 readonly string m_arrayItemName;
39 readonly XmlNameTable m_nameTable;
40
41 public JSONXmlReader(JSONParser parser, JSONXmlReaderOptions options) {
40 42 Safe.ArgumentNotNull(parser, "parser");
41 43 m_parser = parser;
44
45 if (options != null) {
46 m_prefix = options.NodesPrefix ?? String.Empty;
47 m_namespaceUri = options.NamespaceURI ?? String.Empty;
48 m_rootName = options.RootName ?? "json";
49 m_flattenArrays = options.FlattenArrays;
50 m_arrayItemName = options.ArrayItemName ?? "item";
51 m_nameTable = options.NameTable ?? new NameTable();
52 } else {
53 m_prefix = String.Empty;
54 m_namespaceUri = String.Empty;
55 m_rootName = "json";
56 m_flattenArrays = false;
57 m_arrayItemName = "item";
58 m_nameTable = new NameTable();
59 }
42 60 }
43 61
44 62 /// <summary>
45 63 /// Always 0, JSON doesn't support attributes
46 64 /// </summary>
47 65 public override int AttributeCount {
48 66 get { return 0; }
49 67 }
50 68
51 69 public override string BaseURI {
52 70 get { return String.Empty; }
53 71 }
54 72
55 73 public override int Depth {
56 74 get {
57 75 return m_localNameStack.Count+m_depthCorrection;
58 76 }
59 77 }
60 78
61 79 public override bool EOF {
62 80 get { return m_parser.EOF; }
63 81 }
64 82
65 83 /// <summary>
66 84 /// Always throws an exception
67 85 /// </summary>
68 86 /// <param name="i"></param>
69 87 /// <returns></returns>
70 88 public override string GetAttribute(int i) {
71 89 throw new ArgumentOutOfRangeException();
72 90 }
73 91
74 92 /// <summary>
75 93 /// Always returns empty string
76 94 /// </summary>
77 95 /// <param name="name"></param>
78 96 /// <param name="namespaceURI"></param>
79 97 /// <returns></returns>
80 98 public override string GetAttribute(string name, string namespaceURI) {
81 99 return String.Empty;
82 100 }
83 101
84 102 /// <summary>
85 103 /// Always returns empty string
86 104 /// </summary>
87 105 /// <param name="name"></param>
88 106 /// <returns></returns>
89 107 public override string GetAttribute(string name) {
90 108 return String.Empty;
91 109 }
92 110
93 111 public override bool IsEmptyElement {
94 112 get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
95 113 }
96 114
97 115 public override string LocalName {
98 116 get { return m_localName.localName; }
99 117 }
100 118
101 119 public override string LookupNamespace(string prefix) {
102 120 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
103 121 return m_namespaceUri;
104 122 else
105 123 return String.Empty;
106 124 }
107 125
108 126 public override bool MoveToAttribute(string name, string ns) {
109 127 return false;
110 128 }
111 129
112 130 public override bool MoveToAttribute(string name) {
113 131 return false;
114 132 }
115 133
116 134 public override bool MoveToElement() {
117 135 return false;
118 136 }
119 137
120 138 public override bool MoveToFirstAttribute() {
121 139 return false;
122 140 }
123 141
124 142 public override bool MoveToNextAttribute() {
125 143 return false;
126 144 }
127 145
128 146 public override XmlNameTable NameTable {
129 147 get { return m_nameTable; }
130 148 }
131 149
132 150 public override string NamespaceURI {
133 151 get { return m_namespaceUri; }
134 152 }
135 153
136 154 public override XmlNodeType NodeType {
137 155 get {
138 156 switch (m_parser.ElementType) {
139 157 case JSONElementType.BeginObject:
140 158 case JSONElementType.BeginArray:
141 159 return XmlNodeType.Element;
142 160 case JSONElementType.EndObject:
143 161 case JSONElementType.EndArray:
144 162 return XmlNodeType.EndElement;
145 163 case JSONElementType.Value:
146 164 switch (m_valueContext) {
147 165 case ValueContext.ElementStart:
148 166 case ValueContext.ElementEmpty:
149 167 return XmlNodeType.Element;
150 168 case ValueContext.ElementValue:
151 169 return XmlNodeType.Text;
152 170 case ValueContext.ElementEnd:
153 171 return XmlNodeType.EndElement;
154 172 default:
155 173 throw new InvalidOperationException();
156 174 }
157 175 default:
158 176 throw new InvalidOperationException();
159 177 }
160 178 }
161 179 }
162 180
163 181 public override string Prefix {
164 182 get { return m_prefix; }
165 183 }
166 184
167 185 public override bool Read() {
168 186 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
169 187 return false;
170 188
171 189 if (m_state == ReadState.Initial)
172 190 m_state = System.Xml.ReadState.Interactive;
173 191
174 192 try {
175 193 switch (m_parser.ElementType) {
176 194 case JSONElementType.Value:
177 195 switch (m_valueContext) {
178 196 case ValueContext.ElementStart:
179 197 SetLocalName(String.Empty);
180 198 m_valueContext = ValueContext.ElementValue;
181 199 return true;
182 200 case ValueContext.ElementValue:
183 201 RestoreLocalName();
184 202 m_valueContext = ValueContext.ElementEnd;
185 203 return true;
186 204 case ValueContext.ElementEmpty:
187 205 case ValueContext.ElementEnd:
188 206 RestoreLocalName();
189 207 break;
190 208 }
191 209 break;
192 210 case JSONElementType.EndArray:
193 211 case JSONElementType.EndObject:
194 212 RestoreLocalName();
195 213 break;
196 214 }
197 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
215 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : m_arrayItemName;
198 216 while (m_parser.Read()) {
199 217 if (!String.IsNullOrEmpty(m_parser.ElementName))
200 218 itemName = m_parser.ElementName;
201 219
202 220 switch (m_parser.ElementType) {
203 221 case JSONElementType.BeginArray:
204 222 if (m_flattenArrays && !m_localName.isArray) {
205 223 m_depthCorrection--;
206 224 SetLocalName(itemName, true);
207 225 continue;
208 226 } else {
209 227 SetLocalName(itemName, true);
210 228 }
211 229 break;
212 230 case JSONElementType.BeginObject:
213 231 SetLocalName(itemName);
214 232 break;
215 233 case JSONElementType.EndArray:
216 234 if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
217 235 RestoreLocalName();
218 236 m_depthCorrection++;
219 237 continue;
220 238 }
221 239 break;
222 240 case JSONElementType.EndObject:
223 241 break;
224 242 case JSONElementType.Value:
225 243 SetLocalName(itemName);
226 244 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
227 245 break;
228 246 default:
229 247 break;
230 248 }
231 249 return true;
232 250 }
233 251
234 252 m_state = System.Xml.ReadState.EndOfFile;
235 253 return false;
236 254 } catch {
237 255 m_state = System.Xml.ReadState.Error;
238 256 throw;
239 257 }
240 258 }
241 259
242 260 public override bool ReadAttributeValue() {
243 261 return false;
244 262 }
245 263
246 264 public override ReadState ReadState {
247 265 get { return m_state; }
248 266 }
249 267
250 268 public override void ResolveEntity() {
251 269 // do nothing
252 270 }
253 271
254 272 public override string Value {
255 273 get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
256 274 }
257 275
258 276 void SetLocalName(string name) {
259 277 m_localNameStack.Push(m_localName);
260 278 m_localName.localName = name;
261 279 m_localName.isArray = false;
262 280 }
263 281
264 282 void SetLocalName(string name, bool isArray) {
265 283 m_localNameStack.Push(m_localName);
266 284 m_localName.localName = name;
267 285 m_localName.isArray = isArray;
268 286 }
269 287
270 288 void RestoreLocalName() {
271 289 m_localName = m_localNameStack.Pop();
272 290 }
273 291
274 292 public override void Close() {
275 293
276 294 }
277 295
278 296 protected override void Dispose(bool disposing) {
279 297 if (disposing) {
280 298 m_parser.Dispose();
281 299 }
282 300 base.Dispose(disposing);
283 301 }
284 302
303
304 public static JSONXmlReader OpenFile(string file, JSONXmlReaderOptions options) {
305 var stream = File.OpenText(file);
306 var parser = new JSONParser(stream, true);
307 return new JSONXmlReader(parser, options);
285 308 }
286 309 }
310 }
Show file before | Show file after | Hide comments
@@ -1,259 +1,260
1 1 using Implab;
2 2 using System;
3 3 using System.Collections.Generic;
4 4 using System.IO;
5 5 using System.Linq;
6 6 using System.Text;
7 7 using System.Threading.Tasks;
8 8
9 9 namespace Implab.Parsing {
10 10 /// <summary>
11 11 /// Базовый класс для разбора потока входных символов на токены.
12 12 /// </summary>
13 13 /// <remarks>
14 14 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
15 15 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
16 16 /// конца токена и допустимости текущего символа.
17 17 /// </remarks>
18 18 public abstract class Scanner : Disposable {
19 19 struct ScannerConfig {
20 20 public DFAStateDescriptior[] states;
21 21 public int[] alphabetMap;
22 22 }
23 23
24 24 Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
25 25
26 26 DFAStateDescriptior[] m_states;
27 27 int[] m_alphabetMap;
28 28
29 29 protected DFAStateDescriptior m_currentState;
30 30 int m_previewCode;
31 31
32 32 protected int m_tokenLen = 0;
33 33 protected int m_tokenOffset;
34 34
35 35 protected char[] m_buffer;
36 36 protected int m_bufferSize;
37 37 protected int m_pointer;
38 38
39 39 TextReader m_reader;
40 40 bool m_disposeReader;
41 41 int m_chunkSize = 1024; // 1k
42 42 int m_limit = 10 * 1024 * 1024; // 10Mb
43 43
44 44 public Scanner(CDFADefinition definition) {
45 45 Safe.ArgumentNotNull(definition, "definition");
46 46
47 47 m_states = definition.States;
48 48 m_alphabetMap = definition.Alphabet.GetTranslationMap();
49 49
50 50 Feed(new char[0]);
51 51 }
52 52
53 53 /// <summary>
54 54 /// Заполняет входными данными буффер.
55 55 /// </summary>
56 56 /// <param name="data">Данные для обработки.</param>
57 57 /// <remarks>Копирование данных не происходит, переданный массив используется в
58 58 /// качестве входного буффера.</remarks>
59 59 public void Feed(char[] data) {
60 60 Safe.ArgumentNotNull(data, "data");
61 61
62 62 Feed(data, data.Length);
63 63 }
64 64
65 65 /// <summary>
66 66 /// Заполняет буффур чтения входными данными.
67 67 /// </summary>
68 68 /// <param name="data">Данные для обработки.</param>
69 69 /// <param name="length">Длина данных для обработки.</param>
70 70 /// <remarks>Копирование данных не происходит, переданный массив используется в
71 71 /// качестве входного буффера.</remarks>
72 72 public void Feed(char[] data, int length) {
73 73 Safe.ArgumentNotNull(data, "data");
74 74 Safe.ArgumentInRange(length, 0, data.Length, "length");
75 75 AssertNotDisposed();
76 76
77 77 m_pointer = -1;
78 78 m_buffer = data;
79 79 m_bufferSize = length;
80 80 Shift();
81 81 }
82 82
83 83 public void Feed(TextReader reader, bool dispose) {
84 84 Safe.ArgumentNotNull(reader, "reader");
85 85 AssertNotDisposed();
86 86
87 87 if (m_reader != null && m_disposeReader)
88 88 m_reader.Dispose();
89 89
90 90 m_reader = reader;
91 91 m_disposeReader = dispose;
92 92 m_pointer = -1;
93 93 m_buffer = new char[m_chunkSize];
94 94 m_bufferSize = 0;
95 95 Shift();
96 96 }
97 97
98 98 /// <summary>
99 99 /// Получает текущий токен в виде строки.
100 100 /// </summary>
101 101 /// <returns></returns>
102 102 protected string GetTokenValue() {
103 103 return new String(m_buffer, m_tokenOffset, m_tokenLen);
104 104 }
105 105
106 106 /// <summary>
107 107 /// Метки текущего токена, которые были назначены в регулярном выражении.
108 108 /// </summary>
109 109 protected int[] TokenTags {
110 110 get {
111 111 return m_currentState.tag;
112 112 }
113 113 }
114 114
115 115 /// <summary>
116 116 /// Признак конца данных
117 117 /// </summary>
118 118 public bool EOF {
119 119 get {
120 120 return m_pointer >= m_bufferSize;
121 121 }
122 122 }
123 123
124 124 /// <summary>
125 125 /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
126 126 /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
127 127 /// котором находится токен.
128 128 /// </summary>
129 129 /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
130 130 protected bool ReadTokenInternal() {
131 131 if (m_pointer >= m_bufferSize)
132 132 return false;
133 133
134 134 m_currentState = m_states[CDFADefinition.INITIAL_STATE];
135 135 m_tokenLen = 0;
136 136 m_tokenOffset = m_pointer;
137 137 int nextState = CDFADefinition.UNREACHEBLE_STATE;
138 138 do {
139 139 nextState = m_currentState.transitions[m_previewCode];
140 140 if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
141 141 if (m_currentState.final)
142 142 return true;
143 143 else
144 144 throw new ParserException(
145 145 String.Format(
146 146 "Unexpected symbol '{0}', at pos {1}",
147 147 m_buffer[m_pointer],
148 148 Position
149 149 )
150 150 );
151 151 } else {
152 152 m_currentState = m_states[nextState];
153 153 m_tokenLen++;
154 154 }
155 155
156 156 } while (Shift());
157 157
158 158 // END OF DATA
159 159 if (!m_currentState.final)
160 160 throw new ParserException("Unexpected end of data");
161 161
162 162 return true;
163 163 }
164 164
165 165
166 166 bool Shift() {
167 167 m_pointer++;
168 168
169 169 if (m_pointer >= m_bufferSize) {
170 return ReadNextChunk();
170 if (!ReadNextChunk())
171 return false;
171 172 }
172 173
173 174 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
174 175
175 176 return true;
176 177 }
177 178
178 179 bool ReadNextChunk() {
179 180 if (m_reader == null)
180 181 return false;
181 182
182 183 // extend buffer if nesessary
183 184 if (m_pointer + m_chunkSize > m_buffer.Length) {
184 185 // trim unused buffer head
185 186 var size = m_tokenLen + m_chunkSize;
186 187 if (size >= m_limit)
187 188 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
188 189 var temp = new char[size];
189 190 Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
190 191 m_pointer -= m_tokenOffset;
191 192 m_bufferSize -= m_tokenOffset;
192 193 m_tokenOffset = 0;
193 194 m_buffer = temp;
194 195 }
195 196
196 197 var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
197 198 if (read == 0)
198 199 return false;
199 200
200 201 m_bufferSize += read;
201 202
202 203 return true;
203 204 }
204 205
205 206 /// <summary>
206 207 /// Позиция сканнера во входном буфере
207 208 /// </summary>
208 209 public int Position {
209 210 get {
210 211 return m_pointer + 1;
211 212 }
212 213 }
213 214
214 215 /// <summary>
215 216 /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
216 217 /// группировки.
217 218 /// </summary>
218 219 /// <param name="states">Таблица состояний нового ДКА</param>
219 220 /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
220 221 protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
221 222 Safe.ArgumentNotNull(states, "dfa");
222 223
223 224 m_defs.Push(new ScannerConfig {
224 225 states = m_states,
225 226 alphabetMap = m_alphabetMap
226 227 });
227 228
228 229 m_states = states;
229 230 m_alphabetMap = alphabet;
230 231
231 232 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
232 233 }
233 234
234 235 /// <summary>
235 236 /// Восстанавливает предыдущей ДКА сканнера.
236 237 /// </summary>
237 238 protected void Restore() {
238 239 if (m_defs.Count == 0)
239 240 throw new InvalidOperationException();
240 241 var prev = m_defs.Pop();
241 242 m_states = prev.states;
242 243 m_alphabetMap = prev.alphabetMap;
243 244 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
244 245 }
245 246
246 247 protected override void Dispose(bool disposing) {
247 248 if (disposing) {
248 249 if (m_reader != null && m_disposeReader)
249 250 m_reader.Dispose();
250 251 m_buffer = null;
251 252 m_bufferSize = 0;
252 253 m_pointer = 0;
253 254 m_tokenLen = 0;
254 255 m_tokenOffset = 0;
255 256 }
256 257 base.Dispose(disposing);
257 258 }
258 259 }
259 260 }
General Comments 0
You need to be logged in to leave comments. Login now