##// END OF EJS Templates
implab » ImplabNet
RSS
code cleanup
cin -
r59:21611344d366 default
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,104 +1,105
1 1 <?xml version="1.0" encoding="utf-8"?>
2 2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 3 <PropertyGroup>
4 4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5 5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6 6 <ProductVersion>10.0.0</ProductVersion>
7 7 <SchemaVersion>2.0</SchemaVersion>
8 8 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
9 9 <OutputType>Library</OutputType>
10 10 <RootNamespace>Implab</RootNamespace>
11 11 <AssemblyName>Implab</AssemblyName>
12 12 </PropertyGroup>
13 13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
14 14 <DebugSymbols>true</DebugSymbols>
15 15 <DebugType>full</DebugType>
16 16 <Optimize>false</Optimize>
17 17 <OutputPath>bin\Debug</OutputPath>
18 18 <DefineConstants>TRACE;DEBUG;</DefineConstants>
19 19 <ErrorReport>prompt</ErrorReport>
20 20 <WarningLevel>4</WarningLevel>
21 21 <ConsolePause>false</ConsolePause>
22 <RunCodeAnalysis>true</RunCodeAnalysis>
22 23 </PropertyGroup>
23 24 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
24 25 <DebugType>full</DebugType>
25 26 <Optimize>true</Optimize>
26 27 <OutputPath>bin\Release</OutputPath>
27 28 <ErrorReport>prompt</ErrorReport>
28 29 <WarningLevel>4</WarningLevel>
29 30 <ConsolePause>false</ConsolePause>
30 31 </PropertyGroup>
31 32 <ItemGroup>
32 33 <Reference Include="System" />
33 34 <Reference Include="System.XML" />
34 35 </ItemGroup>
35 36 <ItemGroup>
36 37 <Compile Include="Component.cs" />
37 38 <Compile Include="CustomEqualityComparer.cs" />
38 39 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
39 40 <Compile Include="Diagnostics\EventText.cs" />
40 41 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
41 42 <Compile Include="Diagnostics\LogChannel.cs" />
42 43 <Compile Include="Diagnostics\LogicalOperation.cs" />
43 44 <Compile Include="Diagnostics\TextFileListener.cs" />
44 45 <Compile Include="Diagnostics\TextListenerBase.cs" />
45 46 <Compile Include="Diagnostics\TraceLog.cs" />
46 47 <Compile Include="Diagnostics\TraceContext.cs" />
47 48 <Compile Include="Diagnostics\TraceEvent.cs" />
48 49 <Compile Include="Diagnostics\TraceEventType.cs" />
49 50 <Compile Include="Disposable.cs" />
50 51 <Compile Include="ICancellable.cs" />
51 52 <Compile Include="IProgressHandler.cs" />
52 53 <Compile Include="IProgressNotifier.cs" />
53 54 <Compile Include="IPromise.cs" />
54 55 <Compile Include="IPromiseBase.cs" />
55 56 <Compile Include="IServiceLocator.cs" />
56 57 <Compile Include="ITaskController.cs" />
57 58 <Compile Include="JSON\JSONElementContext.cs" />
58 59 <Compile Include="JSON\JSONElementType.cs" />
59 60 <Compile Include="JSON\JSONGrammar.cs" />
60 61 <Compile Include="JSON\JSONParser.cs" />
61 62 <Compile Include="JSON\JSONScanner.cs" />
62 63 <Compile Include="JSON\JsonTokenType.cs" />
63 64 <Compile Include="JSON\JSONWriter.cs" />
64 65 <Compile Include="JSON\JSONXmlReader.cs" />
65 66 <Compile Include="JSON\StringTranslator.cs" />
66 67 <Compile Include="Parallels\DispatchPool.cs" />
67 68 <Compile Include="Parallels\ArrayTraits.cs" />
68 69 <Compile Include="Parallels\MTQueue.cs" />
69 70 <Compile Include="Parallels\WorkerPool.cs" />
70 71 <Compile Include="Parsing\Alphabet.cs" />
71 72 <Compile Include="Parsing\AlphabetBase.cs" />
72 73 <Compile Include="Parsing\AltToken.cs" />
73 74 <Compile Include="Parsing\BinaryToken.cs" />
74 75 <Compile Include="Parsing\CatToken.cs" />
75 76 <Compile Include="Parsing\CDFADefinition.cs" />
76 77 <Compile Include="Parsing\DFABuilder.cs" />
77 78 <Compile Include="Parsing\DFADefinitionBase.cs" />
78 79 <Compile Include="Parsing\DFAStateDescriptor.cs" />
79 80 <Compile Include="Parsing\DFAutomaton.cs" />
80 81 <Compile Include="Parsing\EDFADefinition.cs" />
81 82 <Compile Include="Parsing\EmptyToken.cs" />
82 83 <Compile Include="Parsing\EndToken.cs" />
83 84 <Compile Include="Parsing\EnumAlphabet.cs" />
84 85 <Compile Include="Parsing\Grammar.cs" />
85 86 <Compile Include="Parsing\IAlphabet.cs" />
86 87 <Compile Include="Parsing\IDFADefinition.cs" />
87 88 <Compile Include="Parsing\IVisitor.cs" />
88 89 <Compile Include="Parsing\ParserException.cs" />
89 90 <Compile Include="Parsing\Scanner.cs" />
90 91 <Compile Include="Parsing\StarToken.cs" />
91 92 <Compile Include="Parsing\SymbolToken.cs" />
92 93 <Compile Include="Parsing\Token.cs" />
93 94 <Compile Include="ServiceLocator.cs" />
94 95 <Compile Include="TaskController.cs" />
95 96 <Compile Include="ProgressInitEventArgs.cs" />
96 97 <Compile Include="Properties\AssemblyInfo.cs" />
97 98 <Compile Include="Promise.cs" />
98 99 <Compile Include="Parallels\AsyncPool.cs" />
99 100 <Compile Include="Safe.cs" />
100 101 <Compile Include="ValueEventArgs.cs" />
101 102 </ItemGroup>
102 103 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
103 104 <ItemGroup />
104 105 </Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,203 +1,255
1 1 using Implab;
2 2 using Implab.Parsing;
3 3 using System;
4 4 using System.Collections.Generic;
5 5 using System.Diagnostics;
6 using System.IO;
6 7 using System.Linq;
7 8 using System.Text;
8 9 using System.Threading.Tasks;
9 10
10 11 namespace Implab.JSON {
11 12 /// <summary>
12 13 /// internal
13 14 /// </summary>
14 15 public struct JSONParserContext {
15 16 public string memberName;
16 17 public JSONElementContext elementContext;
17 18 }
18 19
19 20 /// <summary>
20 21 /// Pull парсер JSON данных.
21 22 /// </summary>
22 public class JSONParser : DFAutomaton<JSONParserContext> {
23 public class JSONParser : DFAutomaton<JSONParserContext>, IDisposable {
23 24
24 25 enum MemberContext {
25 26 MemberName,
26 27 MemberValue
27 }
28 }
28 29
29 30 static readonly EnumAlphabet<JsonTokenType> _alphabet = EnumAlphabet<JsonTokenType>.FullAlphabet;
30 31 static readonly DFAStateDescriptior[] _jsonDFA;
31 32 static readonly DFAStateDescriptior[] _objectDFA;
32 33 static readonly DFAStateDescriptior[] _arrayDFA;
33 34
34 35 static JSONParser() {
35 36 var jsonExpression = Token.New(JsonTokenType.BeginObject, JsonTokenType.BeginArray).Tag(0);
36 37
37 38 var valueExpression = Token.New(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
38 39 var memberExpression = Token.New(JsonTokenType.String).Cat(Token.New(JsonTokenType.NameSeparator)).Cat(valueExpression);
39 40 var objectExpression = memberExpression
40 41 .Cat(
41 42 Token.New(JsonTokenType.ValueSeparator)
42 43 .Cat(memberExpression)
43 44 .EClosure()
44 45 )
45 46 .Optional()
46 47 .Cat(Token.New(JsonTokenType.EndObject))
47 48 .Tag(0);
48 49 var arrayExpression = valueExpression
49 50 .Cat(
50 51 Token.New(JsonTokenType.ValueSeparator)
51 52 .Cat(valueExpression)
52 53 .EClosure()
53 54 )
54 55 .Optional()
55 56 .Cat(Token.New(JsonTokenType.EndArray))
56 57 .Tag(0);
57 58
58 59 _jsonDFA = BuildDFA(jsonExpression).States;
59 60 _objectDFA = BuildDFA(objectExpression).States;
60 61 _arrayDFA = BuildDFA(arrayExpression).States;
61 62 }
62 63
63 64 static EDFADefinition<JsonTokenType> BuildDFA(Token expr) {
64 65 var builder = new DFABuilder();
65 66 var dfa = new EDFADefinition<JsonTokenType>(_alphabet);
66 67 expr.Accept(builder);
67 68
68 69 builder.BuildDFA(dfa);
69 70 return dfa;
70 71 }
71 72
72 73 JSONScanner m_scanner;
73 74 MemberContext m_memberContext;
74 75
75 76 JSONElementType m_elementType;
76 77 object m_elementValue;
77 78
79 /// <summary>
80 /// Создает новый парсер на основе строки, содержащей JSON
81 /// </summary>
82 /// <param name="text"></param>
78 83 public JSONParser(string text)
79 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty } ) {
84 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
80 85 Safe.ArgumentNotEmpty(text, "text");
81 86 m_scanner = new JSONScanner();
82 87 m_scanner.Feed(text.ToCharArray());
83 88 }
84 89
90 /// <summary>
91 /// Создает новый экземпляр парсера, на основе текстового потока.
92 /// </summary>
93 /// <param name="reader">Текстовый поток.</param>
94 /// <param name="dispose">Признак того, что парсер должен конролировать время жизни входного потока.</param>
95 public JSONParser(TextReader reader, bool dispose)
96 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
97 Safe.ArgumentNotNull(reader, "reader");
98 m_scanner = new JSONScanner();
99 m_scanner.Feed(reader, dispose);
100 }
101
102 /// <summary>
103 /// Тип текущего элемента на котором стоит парсер.
104 /// </summary>
85 105 public JSONElementType ElementType {
86 106 get { return m_elementType; }
87 107 }
88 108
109 /// <summary>
110 /// Имя элемента - имя свойства родительского контейнера. Для элементов массивов и корневого всегда
111 /// пустая строка.
112 /// </summary>
89 113 public string ElementName {
90 114 get { return m_context.info.memberName; }
91 115 }
92 116
117 /// <summary>
118 /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c>
119 /// </summary>
93 120 public object ElementValue {
94 121 get { return m_elementValue; }
95 122 }
96 123
124 /// <summary>
125 /// Читает слеюудущий объект из потока
126 /// </summary>
127 /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
97 128 public bool Read() {
98 129 if (m_context.current == UNREACHEBLE_STATE)
99 130 throw new InvalidOperationException("The parser is in invalid state");
100 131 object tokenValue;
101 132 JsonTokenType tokenType;
102 133 m_context.info.memberName = String.Empty;
103 134 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
104 135 Move((int)tokenType);
105 136 if (m_context.current == UNREACHEBLE_STATE)
106 137 UnexpectedToken(tokenValue, tokenType);
107 138 switch (tokenType) {
108 139 case JsonTokenType.BeginObject:
109 140 Switch(
110 141 _objectDFA,
111 142 INITIAL_STATE,
112 new JSONParserContext {
143 new JSONParserContext {
113 144 memberName = m_context.info.memberName,
114 145 elementContext = JSONElementContext.Object
115 146 }
116 147 );
117 148 m_elementValue = null;
118 149 m_memberContext = MemberContext.MemberName;
119 150 m_elementType = JSONElementType.BeginObject;
120 151 return true;
121 152 case JsonTokenType.EndObject:
122 153 Restore();
123 154 m_elementValue = null;
124 155 m_elementType = JSONElementType.EndObject;
125 156 return true;
126 157 case JsonTokenType.BeginArray:
127 158 Switch(
128 159 _arrayDFA,
129 160 INITIAL_STATE,
130 161 new JSONParserContext {
131 162 memberName = m_context.info.memberName,
132 163 elementContext = JSONElementContext.Array
133 164 }
134 165 );
135 166 m_elementValue = null;
136 167 m_memberContext = MemberContext.MemberValue;
137 168 m_elementType = JSONElementType.BeginArray;
138 169 return true;
139 170 case JsonTokenType.EndArray:
140 171 Restore();
141 172 m_elementValue = null;
142 173 m_elementType = JSONElementType.EndArray;
143 174 return true;
144 175 case JsonTokenType.String:
145 176 if (m_memberContext == MemberContext.MemberName) {
146 177 m_context.info.memberName = (string)tokenValue;
147 178 break;
148 179 } else {
149 180 m_elementType = JSONElementType.Value;
150 181 m_elementValue = tokenValue;
151 182 return true;
152 183 }
153 184 case JsonTokenType.Number:
154 185 m_elementType = JSONElementType.Value;
155 186 m_elementValue = tokenValue;
156 187 return true;
157 188 case JsonTokenType.Literal:
158 189 m_elementType = JSONElementType.Value;
159 190 m_elementValue = ParseLiteral((string)tokenValue);
160 191 return true;
161 192 case JsonTokenType.NameSeparator:
162 193 m_memberContext = MemberContext.MemberValue;
163 194 break;
164 195 case JsonTokenType.ValueSeparator:
165 196 m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
166 197 break;
167 198 default:
168 199 UnexpectedToken(tokenValue, tokenType);
169 200 break;
170 201 }
171 202 }
172 203 if (m_context.info.elementContext != JSONElementContext.None)
173 204 throw new ParserException("Unexpedted end of data");
174 205 return false;
175 206 }
176 207
177 208 object ParseLiteral(string literal) {
178 209 switch (literal) {
179 210 case "null":
180 211 return null;
181 case "false" :
212 case "false":
182 213 return false;
183 214 case "true":
184 215 return true;
185 216 default:
186 217 UnexpectedToken(literal, JsonTokenType.Literal);
187 218 return null; // avoid compliler error
188 219 }
189 220 }
190 221
191 222 void UnexpectedToken(object value, JsonTokenType tokenType) {
192 223 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
193 224 }
194 225
195 226
227 /// <summary>
228 /// Признак конца потока
229 /// </summary>
196 230 public bool EOF {
197 231 get {
198 232 return m_scanner.EOF;
199 233 }
200 234 }
235
236 protected virtual void Dispose(bool disposing) {
237 if (disposing) {
238 m_scanner.Dispose();
239 }
240 }
241
242 /// <summary>
243 /// Освобождает парсер и связанный с ним сканнер.
244 /// </summary>
245 public void Dispose() {
246 Dispose(true);
247 GC.SuppressFinalize(this);
248 }
249
250 ~JSONParser() {
251 Dispose(false);
252 }
201 253 }
202 254
203 255 }
Show file before | Show file after | Hide comments
@@ -1,278 +1,286
1 1 using Implab;
2 2 using Implab.JSON;
3 3 using Implab.Parsing;
4 4 using System;
5 5 using System.Collections.Generic;
6 6 using System.Linq;
7 7 using System.Text;
8 8 using System.Threading.Tasks;
9 9 using System.Xml;
10 10
11 11 namespace ConsPlay {
12 12 public class JSONXmlReader : XmlReader {
13 13
14 14 enum ValueContext {
15 15 Undefined,
16 16 ElementStart,
17 17 ElementValue,
18 18 ElementEnd,
19 19 ElementEmpty
20 20 }
21 21
22 22 struct LocalNameContext {
23 23 public string localName;
24 24 public bool isArray;
25 25 }
26 26
27 27 JSONParser m_parser;
28 28 ValueContext m_valueContext;
29 29 ReadState m_state = ReadState.Initial;
30 30 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
31 31 LocalNameContext m_localName;
32 32 string m_rootName = "json";
33 33 string m_prefix = String.Empty;
34 34 string m_namespaceUri = String.Empty;
35 35 bool m_flattenArrays = false;
36 36 NameTable m_nameTable = new NameTable();
37 37 int m_depthCorrection = 0;
38 38
39 39 public JSONXmlReader(JSONParser parser) {
40 40 Safe.ArgumentNotNull(parser, "parser");
41 41 m_parser = parser;
42 42 }
43 43
44 44 /// <summary>
45 45 /// Always 0, JSON doesn't support attributes
46 46 /// </summary>
47 47 public override int AttributeCount {
48 48 get { return 0; }
49 49 }
50 50
51 51 public override string BaseURI {
52 52 get { return String.Empty; }
53 53 }
54 54
55 55 public override int Depth {
56 56 get {
57 57 return m_localNameStack.Count+m_depthCorrection;
58 58 }
59 59 }
60 60
61 61 public override bool EOF {
62 62 get { return m_parser.EOF; }
63 63 }
64 64
65 65 /// <summary>
66 66 /// Always throws an exception
67 67 /// </summary>
68 68 /// <param name="i"></param>
69 69 /// <returns></returns>
70 70 public override string GetAttribute(int i) {
71 71 throw new ArgumentOutOfRangeException();
72 72 }
73 73
74 74 /// <summary>
75 75 /// Always returns empty string
76 76 /// </summary>
77 77 /// <param name="name"></param>
78 78 /// <param name="namespaceURI"></param>
79 79 /// <returns></returns>
80 80 public override string GetAttribute(string name, string namespaceURI) {
81 81 return String.Empty;
82 82 }
83 83
84 84 /// <summary>
85 85 /// Always returns empty string
86 86 /// </summary>
87 87 /// <param name="name"></param>
88 88 /// <returns></returns>
89 89 public override string GetAttribute(string name) {
90 90 return String.Empty;
91 91 }
92 92
93 93 public override bool IsEmptyElement {
94 94 get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
95 95 }
96 96
97 97 public override string LocalName {
98 98 get { return m_localName.localName; }
99 99 }
100 100
101 101 public override string LookupNamespace(string prefix) {
102 102 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
103 103 return m_namespaceUri;
104 104 else
105 105 return String.Empty;
106 106 }
107 107
108 108 public override bool MoveToAttribute(string name, string ns) {
109 109 return false;
110 110 }
111 111
112 112 public override bool MoveToAttribute(string name) {
113 113 return false;
114 114 }
115 115
116 116 public override bool MoveToElement() {
117 117 return false;
118 118 }
119 119
120 120 public override bool MoveToFirstAttribute() {
121 121 return false;
122 122 }
123 123
124 124 public override bool MoveToNextAttribute() {
125 125 return false;
126 126 }
127 127
128 128 public override XmlNameTable NameTable {
129 129 get { return m_nameTable; }
130 130 }
131 131
132 132 public override string NamespaceURI {
133 133 get { return m_namespaceUri; }
134 134 }
135 135
136 136 public override XmlNodeType NodeType {
137 137 get {
138 138 switch (m_parser.ElementType) {
139 139 case JSONElementType.BeginObject:
140 140 case JSONElementType.BeginArray:
141 141 return XmlNodeType.Element;
142 142 case JSONElementType.EndObject:
143 143 case JSONElementType.EndArray:
144 144 return XmlNodeType.EndElement;
145 145 case JSONElementType.Value:
146 146 switch (m_valueContext) {
147 147 case ValueContext.ElementStart:
148 148 case ValueContext.ElementEmpty:
149 149 return XmlNodeType.Element;
150 150 case ValueContext.ElementValue:
151 151 return XmlNodeType.Text;
152 152 case ValueContext.ElementEnd:
153 153 return XmlNodeType.EndElement;
154 154 default:
155 155 throw new InvalidOperationException();
156 156 }
157 157 default:
158 158 throw new InvalidOperationException();
159 159 }
160 160 }
161 161 }
162 162
163 163 public override string Prefix {
164 164 get { return m_prefix; }
165 165 }
166 166
167 167 public override bool Read() {
168 168 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
169 169 return false;
170 170
171 171 if (m_state == ReadState.Initial)
172 172 m_state = System.Xml.ReadState.Interactive;
173 173
174 174 try {
175 175 switch (m_parser.ElementType) {
176 176 case JSONElementType.Value:
177 177 switch (m_valueContext) {
178 178 case ValueContext.ElementStart:
179 179 SetLocalName(String.Empty);
180 180 m_valueContext = ValueContext.ElementValue;
181 181 return true;
182 182 case ValueContext.ElementValue:
183 183 RestoreLocalName();
184 184 m_valueContext = ValueContext.ElementEnd;
185 185 return true;
186 186 case ValueContext.ElementEmpty:
187 187 case ValueContext.ElementEnd:
188 188 RestoreLocalName();
189 189 break;
190 190 }
191 191 break;
192 192 case JSONElementType.EndArray:
193 193 case JSONElementType.EndObject:
194 194 RestoreLocalName();
195 195 break;
196 196 }
197 197 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
198 198 while (m_parser.Read()) {
199 199 if (!String.IsNullOrEmpty(m_parser.ElementName))
200 200 itemName = m_parser.ElementName;
201 201
202 202 switch (m_parser.ElementType) {
203 203 case JSONElementType.BeginArray:
204 204 if (m_flattenArrays && !m_localName.isArray) {
205 205 m_depthCorrection--;
206 206 SetLocalName(itemName, true);
207 207 continue;
208 208 } else {
209 209 SetLocalName(itemName, true);
210 210 }
211 211 break;
212 212 case JSONElementType.BeginObject:
213 213 SetLocalName(itemName);
214 214 break;
215 215 case JSONElementType.EndArray:
216 216 if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
217 217 RestoreLocalName();
218 218 m_depthCorrection++;
219 219 continue;
220 220 }
221 221 break;
222 222 case JSONElementType.EndObject:
223 223 break;
224 224 case JSONElementType.Value:
225 225 SetLocalName(itemName);
226 226 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
227 227 break;
228 228 default:
229 229 break;
230 230 }
231 231 return true;
232 232 }
233 233
234 234 m_state = System.Xml.ReadState.EndOfFile;
235 235 return false;
236 236 } catch {
237 237 m_state = System.Xml.ReadState.Error;
238 238 throw;
239 239 }
240 240 }
241 241
242 242 public override bool ReadAttributeValue() {
243 243 return false;
244 244 }
245 245
246 246 public override ReadState ReadState {
247 247 get { return m_state; }
248 248 }
249 249
250 250 public override void ResolveEntity() {
251 251 // do nothing
252 252 }
253 253
254 254 public override string Value {
255 255 get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
256 256 }
257 257
258 258 void SetLocalName(string name) {
259 259 m_localNameStack.Push(m_localName);
260 260 m_localName.localName = name;
261 261 m_localName.isArray = false;
262 262 }
263 263
264 264 void SetLocalName(string name, bool isArray) {
265 265 m_localNameStack.Push(m_localName);
266 266 m_localName.localName = name;
267 267 m_localName.isArray = isArray;
268 268 }
269 269
270 270 void RestoreLocalName() {
271 271 m_localName = m_localNameStack.Pop();
272 272 }
273 273
274 274 public override void Close() {
275 m_state = System.Xml.ReadState.EndOfFile ;
275
276 276 }
277
278 protected override void Dispose(bool disposing) {
279 if (disposing) {
280 m_parser.Dispose();
281 }
282 base.Dispose(disposing);
283 }
284
277 285 }
278 286 }
Show file before | Show file after | Hide comments
@@ -1,23 +1,23
1 1 using Implab;
2 2 using System;
3 3 using System.Collections.Generic;
4 4 using System.Linq;
5 5 using System.Text;
6 6 using System.Threading.Tasks;
7 7
8 8 namespace Implab.Parsing {
9 9 public class Alphabet: AlphabetBase<char> {
10 10
11 public Alphabet()
12 : base(char.MaxValue + 1) {
13 }
14
11 15 public override int GetSymbolIndex(char symbol) {
12 16 return symbol;
13 17 }
14 18
15 19 public override IEnumerable<char> InputSymbols {
16 20 get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); }
17 21 }
18
19 protected override int MapSize {
20 get { return char.MaxValue + 1; }
21 }
22 22 }
23 23 }
Show file before | Show file after | Hide comments
@@ -1,103 +1,100
1 1 using Implab;
2 2 using System;
3 3 using System.Collections.Generic;
4 4 using System.Diagnostics;
5 5 using System.Linq;
6 6 using System.Text;
7 7 using System.Threading.Tasks;
8 8
9 9 namespace Implab.Parsing {
10 10 public abstract class AlphabetBase<T> : IAlphabet<T> {
11 11 public const int UNCLASSIFIED = 0;
12 12
13 13 int m_nextId = 1;
14 14 int[] m_map;
15 15
16 16 public int Count {
17 17 get { return m_nextId; }
18 18 }
19 19
20 protected AlphabetBase() {
21 m_map = new int[MapSize];
20 protected AlphabetBase(int mapSize) {
21 m_map = new int[mapSize];
22 22 }
23 23
24 24 protected AlphabetBase(int[] map) {
25 25 Debug.Assert(map != null);
26 Debug.Assert(map.Length == MapSize);
27 26
28 27 m_map = map;
29 28 m_nextId = map.Max() + 1;
30 29 }
31 30
32 31 public int DefineSymbol(T symbol) {
33 32 var index = GetSymbolIndex(symbol);
34 33 if (m_map[index] == UNCLASSIFIED)
35 34 m_map[index] = m_nextId++;
36 35 return m_map[index];
37 36 }
38 37
39 38 public int DefineClass(IEnumerable<T> symbols) {
40 39 Safe.ArgumentNotNull(symbols, "symbols");
41 40 symbols = symbols.Distinct();
42 41
43 42 foreach (var symbol in symbols) {
44 43 var index = GetSymbolIndex(symbol);
45 44 if (m_map[index] == UNCLASSIFIED)
46 45 m_map[GetSymbolIndex(symbol)] = m_nextId;
47 46 else
48 47 throw new InvalidOperationException(String.Format("Symbol '{0}' already in use", symbol));
49 48 }
50 49 return m_nextId++;
51 50 }
52 51
53 52 public List<T>[] CreateReverseMap() {
54 53 return
55 54 Enumerable.Range(UNCLASSIFIED, Count)
56 55 .Select(
57 56 i => InputSymbols
58 57 .Where(x => i != UNCLASSIFIED && m_map[GetSymbolIndex(x)] == i)
59 58 .ToList()
60 59 )
61 60 .ToArray();
62 61 }
63 62
64 63 public int[] Reclassify(IAlphabet<T> newAlphabet, IEnumerable<ICollection<int>> classes) {
65 64 Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
66 65 Safe.ArgumentNotNull(classes, "classes");
67 66 var reverseMap = CreateReverseMap();
68 67
69 68 int[] translationMap = new int[Count];
70 69
71 70 foreach (var scl in classes) {
72 71 // skip if the supper class contains the unclassified element
73 72 if (scl.Contains(UNCLASSIFIED))
74 73 continue;
75 74 var range = new List<T>();
76 75 foreach (var cl in scl) {
77 76 if (cl < 0 || cl >= reverseMap.Length)
78 77 throw new ArgumentOutOfRangeException(String.Format("Class {0} is not valid for the current alphabet", cl));
79 78 range.AddRange(reverseMap[cl]);
80 79 }
81 80 var newClass = newAlphabet.DefineClass(range);
82 81 foreach (var cl in scl)
83 82 translationMap[cl] = newClass;
84 83 }
85 84
86 85 return translationMap;
87 86 }
88 87
89 88 public int Translate(T symbol) {
90 89 return m_map[GetSymbolIndex(symbol)];
91 90 }
92 91
93 92 public abstract int GetSymbolIndex(T symbol);
94 93
95 94 public abstract IEnumerable<T> InputSymbols { get; }
96 95
97 protected abstract int MapSize { get; }
98
99 96 public int[] GetTranslationMap() {
100 97 return m_map;
101 98 }
102 99 }
103 100 }
Show file before | Show file after | Hide comments
@@ -1,68 +1,67
1 1 using Implab;
2 2 using System;
3 3 using System.Collections.Generic;
4 using System.Diagnostics;
4 5 using System.Globalization;
5 6 using System.Linq;
6 7 using System.Text;
7 8 using System.Threading.Tasks;
8 9
9 10 namespace Implab.Parsing {
10 11 /// <summary>
11 12 /// Алфавит символами которого являются элементы перечислений.
12 13 /// </summary>
13 14 /// <typeparam name="T">Тип перечислений</typeparam>
14 15 public class EnumAlphabet<T> : AlphabetBase<T> where T : struct, IConvertible {
15 16 static readonly T[] _symbols;
16 17 static readonly EnumAlphabet<T> _fullAlphabet;
17 18
18 19 static EnumAlphabet() {
19 20 if (!typeof(T).IsEnum)
20 21 throw new InvalidOperationException("Invalid generic parameter, enumeration is required");
21 22
22 23 if (Enum.GetUnderlyingType(typeof(T)) != typeof(Int32))
23 24 throw new InvalidOperationException("Only enums based on Int32 are supported");
24 25
25 26 _symbols = ((T[])Enum.GetValues(typeof(T)))
26 27 .OrderBy(x => x.ToInt32(CultureInfo.InvariantCulture))
27 28 .ToArray();
28 29
29 30 if (
30 31 _symbols[_symbols.Length - 1].ToInt32(CultureInfo.InvariantCulture) >= _symbols.Length
31 32 || _symbols[0].ToInt32(CultureInfo.InvariantCulture) != 0
32 33 )
33 34 throw new InvalidOperationException("The specified enumeration must be zero-based and continuously numbered");
34 35
35 36 _fullAlphabet = new EnumAlphabet<T>(_symbols.Select(x => x.ToInt32(CultureInfo.InvariantCulture)).ToArray());
36 37 }
37 38
38 39
39 40
40 41 public static EnumAlphabet<T> FullAlphabet {
41 42 get {
42 43 return _fullAlphabet;
43 44 }
44 45 }
45 46
46 47
47 48 public EnumAlphabet()
48 : base() {
49 : base(_symbols.Length) {
49 50 }
50 51
51 52 public EnumAlphabet(int[] map)
52 53 : base(map) {
54 Debug.Assert(map.Length == _symbols.Length);
53 55 }
54 56
55 57
56 58 public override int GetSymbolIndex(T symbol) {
57 59 return symbol.ToInt32(CultureInfo.InvariantCulture);
58 60 }
59 61
60 62 public override IEnumerable<T> InputSymbols {
61 63 get { return _symbols; }
62 64 }
63 65
64 protected override int MapSize {
65 get { return _symbols.Length; }
66 }
67 66 }
68 67 }
Show file before | Show file after | Hide comments
@@ -1,216 +1,259
1 1 using Implab;
2 2 using System;
3 3 using System.Collections.Generic;
4 using System.IO;
4 5 using System.Linq;
5 6 using System.Text;
6 7 using System.Threading.Tasks;
7 8
8 9 namespace Implab.Parsing {
9 10 /// <summary>
10 11 /// Базовый класс для разбора потока входных символов на токены.
11 12 /// </summary>
12 13 /// <remarks>
13 14 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
14 15 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
15 16 /// конца токена и допустимости текущего символа.
16 17 /// </remarks>
17 public class Scanner {
18 public abstract class Scanner : Disposable {
18 19 struct ScannerConfig {
19 20 public DFAStateDescriptior[] states;
20 21 public int[] alphabetMap;
21 22 }
22 23
23 24 Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
24 25
25 26 DFAStateDescriptior[] m_states;
26 27 int[] m_alphabetMap;
27 28
28 29 protected DFAStateDescriptior m_currentState;
29 30 int m_previewCode;
30 31
31 32 protected int m_tokenLen = 0;
32 33 protected int m_tokenOffset;
33 34
34 35 protected char[] m_buffer;
35 36 protected int m_bufferSize;
36 37 protected int m_pointer;
37 38
38 public Scanner(CDFADefinition definition, string text) {
39 Safe.ArgumentNotNull(definition, "definition");
40 Safe.ArgumentNotEmpty(text, "text");
41
42 m_states = definition.States;
43 m_alphabetMap = definition.Alphabet.GetTranslationMap();
44
45 Feed(text.ToCharArray());
46 }
39 TextReader m_reader;
40 bool m_disposeReader;
41 int m_chunkSize = 1024; // 1k
42 int m_limit = 10 * 1024 * 1024; // 10Mb
47 43
48 44 public Scanner(CDFADefinition definition) {
49 45 Safe.ArgumentNotNull(definition, "definition");
50 46
51 47 m_states = definition.States;
52 48 m_alphabetMap = definition.Alphabet.GetTranslationMap();
53 49
54 50 Feed(new char[0]);
55 51 }
56 52
57 53 /// <summary>
58 54 /// Заполняет входными данными буффер.
59 55 /// </summary>
60 56 /// <param name="data">Данные для обработки.</param>
61 57 /// <remarks>Копирование данных не происходит, переданный массив используется в
62 58 /// качестве входного буффера.</remarks>
63 59 public void Feed(char[] data) {
64 60 Safe.ArgumentNotNull(data, "data");
65 61
66 62 Feed(data, data.Length);
67 63 }
68 64
69 65 /// <summary>
70 66 /// Заполняет буффур чтения входными данными.
71 67 /// </summary>
72 68 /// <param name="data">Данные для обработки.</param>
73 69 /// <param name="length">Длина данных для обработки.</param>
74 70 /// <remarks>Копирование данных не происходит, переданный массив используется в
75 71 /// качестве входного буффера.</remarks>
76 72 public void Feed(char[] data, int length) {
77 73 Safe.ArgumentNotNull(data, "data");
78 74 Safe.ArgumentInRange(length, 0, data.Length, "length");
75 AssertNotDisposed();
79 76
80 77 m_pointer = -1;
81 78 m_buffer = data;
82 79 m_bufferSize = length;
83 80 Shift();
84 81 }
85 82
83 public void Feed(TextReader reader, bool dispose) {
84 Safe.ArgumentNotNull(reader, "reader");
85 AssertNotDisposed();
86
87 if (m_reader != null && m_disposeReader)
88 m_reader.Dispose();
89
90 m_reader = reader;
91 m_disposeReader = dispose;
92 m_pointer = -1;
93 m_buffer = new char[m_chunkSize];
94 m_bufferSize = 0;
95 Shift();
96 }
97
86 98 /// <summary>
87 99 /// Получает текущий токен в виде строки.
88 100 /// </summary>
89 101 /// <returns></returns>
90 public string GetTokenValue() {
102 protected string GetTokenValue() {
91 103 return new String(m_buffer, m_tokenOffset, m_tokenLen);
92 104 }
93 105
94 106 /// <summary>
95 107 /// Метки текущего токена, которые были назначены в регулярном выражении.
96 108 /// </summary>
97 public int[] TokenTags {
109 protected int[] TokenTags {
98 110 get {
99 111 return m_currentState.tag;
100 112 }
101 113 }
102 114
103 115 /// <summary>
104 116 /// Признак конца данных
105 117 /// </summary>
106 118 public bool EOF {
107 119 get {
108 120 return m_pointer >= m_bufferSize;
109 121 }
110 122 }
111 123
112 124 /// <summary>
113 125 /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
114 126 /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
115 127 /// котором находится токен.
116 128 /// </summary>
117 129 /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
118 130 protected bool ReadTokenInternal() {
119 131 if (m_pointer >= m_bufferSize)
120 132 return false;
121 133
122 134 m_currentState = m_states[CDFADefinition.INITIAL_STATE];
123 135 m_tokenLen = 0;
124 136 m_tokenOffset = m_pointer;
125 137 int nextState = CDFADefinition.UNREACHEBLE_STATE;
126 138 do {
127 139 nextState = m_currentState.transitions[m_previewCode];
128 140 if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
129 141 if (m_currentState.final)
130 142 return true;
131 143 else
132 144 throw new ParserException(
133 145 String.Format(
134 146 "Unexpected symbol '{0}', at pos {1}",
135 147 m_buffer[m_pointer],
136 148 Position
137 149 )
138 150 );
139 151 } else {
140 152 m_currentState = m_states[nextState];
141 153 m_tokenLen++;
142 154 }
143 155
144 156 } while (Shift());
145 157
146 158 // END OF DATA
147 159 if (!m_currentState.final)
148 160 throw new ParserException("Unexpected end of data");
149 161
150 162 return true;
151 163 }
152 164
153 165
154 166 bool Shift() {
155 167 m_pointer++;
156 168
157 169 if (m_pointer >= m_bufferSize) {
158 170 return ReadNextChunk();
159 171 }
160 172
161 173 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
162 174
163 175 return true;
164 176 }
165 177
166 /// <summary>
167 /// Вызывается по достижению конца входного буффера для получения
168 /// новых данных.
169 /// </summary>
170 /// <returns><c>true</c> - новые двнные получены, можно продолжать обработку.</returns>
171 protected virtual bool ReadNextChunk() {
172 return false;
178 bool ReadNextChunk() {
179 if (m_reader == null)
180 return false;
181
182 // extend buffer if nesessary
183 if (m_pointer + m_chunkSize > m_buffer.Length) {
184 // trim unused buffer head
185 var size = m_tokenLen + m_chunkSize;
186 if (size >= m_limit)
187 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
188 var temp = new char[size];
189 Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
190 m_pointer -= m_tokenOffset;
191 m_bufferSize -= m_tokenOffset;
192 m_tokenOffset = 0;
193 m_buffer = temp;
194 }
195
196 var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
197 if (read == 0)
198 return false;
199
200 m_bufferSize += read;
201
202 return true;
173 203 }
174 204
175 205 /// <summary>
176 206 /// Позиция сканнера во входном буфере
177 207 /// </summary>
178 208 public int Position {
179 209 get {
180 210 return m_pointer + 1;
181 211 }
182 212 }
183 213
184 214 /// <summary>
185 215 /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
186 216 /// группировки.
187 217 /// </summary>
188 218 /// <param name="states">Таблица состояний нового ДКА</param>
189 219 /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
190 220 protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
191 221 Safe.ArgumentNotNull(states, "dfa");
192 222
193 223 m_defs.Push(new ScannerConfig {
194 224 states = m_states,
195 225 alphabetMap = m_alphabetMap
196 226 });
197 227
198 228 m_states = states;
199 229 m_alphabetMap = alphabet;
200 230
201 231 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
202 232 }
203 233
204 234 /// <summary>
205 235 /// Восстанавливает предыдущей ДКА сканнера.
206 236 /// </summary>
207 237 protected void Restore() {
208 238 if (m_defs.Count == 0)
209 239 throw new InvalidOperationException();
210 240 var prev = m_defs.Pop();
211 241 m_states = prev.states;
212 242 m_alphabetMap = prev.alphabetMap;
213 243 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
214 244 }
245
246 protected override void Dispose(bool disposing) {
247 if (disposing) {
248 if (m_reader != null && m_disposeReader)
249 m_reader.Dispose();
250 m_buffer = null;
251 m_bufferSize = 0;
252 m_pointer = 0;
253 m_tokenLen = 0;
254 m_tokenOffset = 0;
255 }
256 base.Dispose(disposing);
257 }
215 258 }
216 259 }
Show file before | Show file after | Hide comments
@@ -1,25 +1,27
1 1 using System.Reflection;
2 using System.Runtime.CompilerServices;
2 using System.Runtime.CompilerServices;
3 using System.Runtime.InteropServices;
3 4
4 5 // Information about this assembly is defined by the following attributes.
5 6 // Change them to the values specific to your project.
6 7
7 8 [assembly: AssemblyTitle("Implab")]
8 9 [assembly: AssemblyDescription("Tools")]
9 10 [assembly: AssemblyConfiguration("")]
10 11 [assembly: AssemblyCompany("")]
11 12 [assembly: AssemblyProduct("")]
12 13 [assembly: AssemblyCopyright("Implab")]
13 14 [assembly: AssemblyTrademark("")]
14 15 // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
15 16 // The form "{Major}.{Minor}.*" will automatically update the build and revision,
16 17 // and "{Major}.{Minor}.{Build}.*" will update just the revision.
17 18
18 19 [assembly: AssemblyVersion("1.0.*")]
20 [assembly: ComVisible(false)]
19 21
20 22 // The following attributes are used to specify the signing key for the assembly,
21 23 // if desired. See the Mono documentation for more information about signing.
22 24
23 25 //[assembly: AssemblyDelaySign(false)]
24 26 //[assembly: AssemblyKeyFile("")]
25 27
General Comments 0
You need to be logged in to leave comments. Login now