##// END OF EJS Templates
code cleanup
cin -
r59:21611344d366 default
parent child
Show More
@@ -1,104 +1,105
1 <?xml version="1.0" encoding="utf-8"?>
1 <?xml version="1.0" encoding="utf-8"?>
2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 <PropertyGroup>
3 <PropertyGroup>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6 <ProductVersion>10.0.0</ProductVersion>
6 <ProductVersion>10.0.0</ProductVersion>
7 <SchemaVersion>2.0</SchemaVersion>
7 <SchemaVersion>2.0</SchemaVersion>
8 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
8 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
9 <OutputType>Library</OutputType>
9 <OutputType>Library</OutputType>
10 <RootNamespace>Implab</RootNamespace>
10 <RootNamespace>Implab</RootNamespace>
11 <AssemblyName>Implab</AssemblyName>
11 <AssemblyName>Implab</AssemblyName>
12 </PropertyGroup>
12 </PropertyGroup>
13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
14 <DebugSymbols>true</DebugSymbols>
14 <DebugSymbols>true</DebugSymbols>
15 <DebugType>full</DebugType>
15 <DebugType>full</DebugType>
16 <Optimize>false</Optimize>
16 <Optimize>false</Optimize>
17 <OutputPath>bin\Debug</OutputPath>
17 <OutputPath>bin\Debug</OutputPath>
18 <DefineConstants>TRACE;DEBUG;</DefineConstants>
18 <DefineConstants>TRACE;DEBUG;</DefineConstants>
19 <ErrorReport>prompt</ErrorReport>
19 <ErrorReport>prompt</ErrorReport>
20 <WarningLevel>4</WarningLevel>
20 <WarningLevel>4</WarningLevel>
21 <ConsolePause>false</ConsolePause>
21 <ConsolePause>false</ConsolePause>
22 <RunCodeAnalysis>true</RunCodeAnalysis>
22 </PropertyGroup>
23 </PropertyGroup>
23 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
24 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
24 <DebugType>full</DebugType>
25 <DebugType>full</DebugType>
25 <Optimize>true</Optimize>
26 <Optimize>true</Optimize>
26 <OutputPath>bin\Release</OutputPath>
27 <OutputPath>bin\Release</OutputPath>
27 <ErrorReport>prompt</ErrorReport>
28 <ErrorReport>prompt</ErrorReport>
28 <WarningLevel>4</WarningLevel>
29 <WarningLevel>4</WarningLevel>
29 <ConsolePause>false</ConsolePause>
30 <ConsolePause>false</ConsolePause>
30 </PropertyGroup>
31 </PropertyGroup>
31 <ItemGroup>
32 <ItemGroup>
32 <Reference Include="System" />
33 <Reference Include="System" />
33 <Reference Include="System.XML" />
34 <Reference Include="System.XML" />
34 </ItemGroup>
35 </ItemGroup>
35 <ItemGroup>
36 <ItemGroup>
36 <Compile Include="Component.cs" />
37 <Compile Include="Component.cs" />
37 <Compile Include="CustomEqualityComparer.cs" />
38 <Compile Include="CustomEqualityComparer.cs" />
38 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
39 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
39 <Compile Include="Diagnostics\EventText.cs" />
40 <Compile Include="Diagnostics\EventText.cs" />
40 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
41 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
41 <Compile Include="Diagnostics\LogChannel.cs" />
42 <Compile Include="Diagnostics\LogChannel.cs" />
42 <Compile Include="Diagnostics\LogicalOperation.cs" />
43 <Compile Include="Diagnostics\LogicalOperation.cs" />
43 <Compile Include="Diagnostics\TextFileListener.cs" />
44 <Compile Include="Diagnostics\TextFileListener.cs" />
44 <Compile Include="Diagnostics\TextListenerBase.cs" />
45 <Compile Include="Diagnostics\TextListenerBase.cs" />
45 <Compile Include="Diagnostics\TraceLog.cs" />
46 <Compile Include="Diagnostics\TraceLog.cs" />
46 <Compile Include="Diagnostics\TraceContext.cs" />
47 <Compile Include="Diagnostics\TraceContext.cs" />
47 <Compile Include="Diagnostics\TraceEvent.cs" />
48 <Compile Include="Diagnostics\TraceEvent.cs" />
48 <Compile Include="Diagnostics\TraceEventType.cs" />
49 <Compile Include="Diagnostics\TraceEventType.cs" />
49 <Compile Include="Disposable.cs" />
50 <Compile Include="Disposable.cs" />
50 <Compile Include="ICancellable.cs" />
51 <Compile Include="ICancellable.cs" />
51 <Compile Include="IProgressHandler.cs" />
52 <Compile Include="IProgressHandler.cs" />
52 <Compile Include="IProgressNotifier.cs" />
53 <Compile Include="IProgressNotifier.cs" />
53 <Compile Include="IPromise.cs" />
54 <Compile Include="IPromise.cs" />
54 <Compile Include="IPromiseBase.cs" />
55 <Compile Include="IPromiseBase.cs" />
55 <Compile Include="IServiceLocator.cs" />
56 <Compile Include="IServiceLocator.cs" />
56 <Compile Include="ITaskController.cs" />
57 <Compile Include="ITaskController.cs" />
57 <Compile Include="JSON\JSONElementContext.cs" />
58 <Compile Include="JSON\JSONElementContext.cs" />
58 <Compile Include="JSON\JSONElementType.cs" />
59 <Compile Include="JSON\JSONElementType.cs" />
59 <Compile Include="JSON\JSONGrammar.cs" />
60 <Compile Include="JSON\JSONGrammar.cs" />
60 <Compile Include="JSON\JSONParser.cs" />
61 <Compile Include="JSON\JSONParser.cs" />
61 <Compile Include="JSON\JSONScanner.cs" />
62 <Compile Include="JSON\JSONScanner.cs" />
62 <Compile Include="JSON\JsonTokenType.cs" />
63 <Compile Include="JSON\JsonTokenType.cs" />
63 <Compile Include="JSON\JSONWriter.cs" />
64 <Compile Include="JSON\JSONWriter.cs" />
64 <Compile Include="JSON\JSONXmlReader.cs" />
65 <Compile Include="JSON\JSONXmlReader.cs" />
65 <Compile Include="JSON\StringTranslator.cs" />
66 <Compile Include="JSON\StringTranslator.cs" />
66 <Compile Include="Parallels\DispatchPool.cs" />
67 <Compile Include="Parallels\DispatchPool.cs" />
67 <Compile Include="Parallels\ArrayTraits.cs" />
68 <Compile Include="Parallels\ArrayTraits.cs" />
68 <Compile Include="Parallels\MTQueue.cs" />
69 <Compile Include="Parallels\MTQueue.cs" />
69 <Compile Include="Parallels\WorkerPool.cs" />
70 <Compile Include="Parallels\WorkerPool.cs" />
70 <Compile Include="Parsing\Alphabet.cs" />
71 <Compile Include="Parsing\Alphabet.cs" />
71 <Compile Include="Parsing\AlphabetBase.cs" />
72 <Compile Include="Parsing\AlphabetBase.cs" />
72 <Compile Include="Parsing\AltToken.cs" />
73 <Compile Include="Parsing\AltToken.cs" />
73 <Compile Include="Parsing\BinaryToken.cs" />
74 <Compile Include="Parsing\BinaryToken.cs" />
74 <Compile Include="Parsing\CatToken.cs" />
75 <Compile Include="Parsing\CatToken.cs" />
75 <Compile Include="Parsing\CDFADefinition.cs" />
76 <Compile Include="Parsing\CDFADefinition.cs" />
76 <Compile Include="Parsing\DFABuilder.cs" />
77 <Compile Include="Parsing\DFABuilder.cs" />
77 <Compile Include="Parsing\DFADefinitionBase.cs" />
78 <Compile Include="Parsing\DFADefinitionBase.cs" />
78 <Compile Include="Parsing\DFAStateDescriptor.cs" />
79 <Compile Include="Parsing\DFAStateDescriptor.cs" />
79 <Compile Include="Parsing\DFAutomaton.cs" />
80 <Compile Include="Parsing\DFAutomaton.cs" />
80 <Compile Include="Parsing\EDFADefinition.cs" />
81 <Compile Include="Parsing\EDFADefinition.cs" />
81 <Compile Include="Parsing\EmptyToken.cs" />
82 <Compile Include="Parsing\EmptyToken.cs" />
82 <Compile Include="Parsing\EndToken.cs" />
83 <Compile Include="Parsing\EndToken.cs" />
83 <Compile Include="Parsing\EnumAlphabet.cs" />
84 <Compile Include="Parsing\EnumAlphabet.cs" />
84 <Compile Include="Parsing\Grammar.cs" />
85 <Compile Include="Parsing\Grammar.cs" />
85 <Compile Include="Parsing\IAlphabet.cs" />
86 <Compile Include="Parsing\IAlphabet.cs" />
86 <Compile Include="Parsing\IDFADefinition.cs" />
87 <Compile Include="Parsing\IDFADefinition.cs" />
87 <Compile Include="Parsing\IVisitor.cs" />
88 <Compile Include="Parsing\IVisitor.cs" />
88 <Compile Include="Parsing\ParserException.cs" />
89 <Compile Include="Parsing\ParserException.cs" />
89 <Compile Include="Parsing\Scanner.cs" />
90 <Compile Include="Parsing\Scanner.cs" />
90 <Compile Include="Parsing\StarToken.cs" />
91 <Compile Include="Parsing\StarToken.cs" />
91 <Compile Include="Parsing\SymbolToken.cs" />
92 <Compile Include="Parsing\SymbolToken.cs" />
92 <Compile Include="Parsing\Token.cs" />
93 <Compile Include="Parsing\Token.cs" />
93 <Compile Include="ServiceLocator.cs" />
94 <Compile Include="ServiceLocator.cs" />
94 <Compile Include="TaskController.cs" />
95 <Compile Include="TaskController.cs" />
95 <Compile Include="ProgressInitEventArgs.cs" />
96 <Compile Include="ProgressInitEventArgs.cs" />
96 <Compile Include="Properties\AssemblyInfo.cs" />
97 <Compile Include="Properties\AssemblyInfo.cs" />
97 <Compile Include="Promise.cs" />
98 <Compile Include="Promise.cs" />
98 <Compile Include="Parallels\AsyncPool.cs" />
99 <Compile Include="Parallels\AsyncPool.cs" />
99 <Compile Include="Safe.cs" />
100 <Compile Include="Safe.cs" />
100 <Compile Include="ValueEventArgs.cs" />
101 <Compile Include="ValueEventArgs.cs" />
101 </ItemGroup>
102 </ItemGroup>
102 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
103 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
103 <ItemGroup />
104 <ItemGroup />
104 </Project> No newline at end of file
105 </Project>
@@ -1,203 +1,255
1 using Implab;
1 using Implab;
2 using Implab.Parsing;
2 using Implab.Parsing;
3 using System;
3 using System;
4 using System.Collections.Generic;
4 using System.Collections.Generic;
5 using System.Diagnostics;
5 using System.Diagnostics;
6 using System.IO;
6 using System.Linq;
7 using System.Linq;
7 using System.Text;
8 using System.Text;
8 using System.Threading.Tasks;
9 using System.Threading.Tasks;
9
10
10 namespace Implab.JSON {
11 namespace Implab.JSON {
11 /// <summary>
12 /// <summary>
12 /// internal
13 /// internal
13 /// </summary>
14 /// </summary>
14 public struct JSONParserContext {
15 public struct JSONParserContext {
15 public string memberName;
16 public string memberName;
16 public JSONElementContext elementContext;
17 public JSONElementContext elementContext;
17 }
18 }
18
19
19 /// <summary>
20 /// <summary>
20 /// Pull парсер JSON данных.
21 /// Pull парсер JSON данных.
21 /// </summary>
22 /// </summary>
22 public class JSONParser : DFAutomaton<JSONParserContext> {
23 public class JSONParser : DFAutomaton<JSONParserContext>, IDisposable {
23
24
24 enum MemberContext {
25 enum MemberContext {
25 MemberName,
26 MemberName,
26 MemberValue
27 MemberValue
27 }
28 }
28
29
29 static readonly EnumAlphabet<JsonTokenType> _alphabet = EnumAlphabet<JsonTokenType>.FullAlphabet;
30 static readonly EnumAlphabet<JsonTokenType> _alphabet = EnumAlphabet<JsonTokenType>.FullAlphabet;
30 static readonly DFAStateDescriptior[] _jsonDFA;
31 static readonly DFAStateDescriptior[] _jsonDFA;
31 static readonly DFAStateDescriptior[] _objectDFA;
32 static readonly DFAStateDescriptior[] _objectDFA;
32 static readonly DFAStateDescriptior[] _arrayDFA;
33 static readonly DFAStateDescriptior[] _arrayDFA;
33
34
34 static JSONParser() {
35 static JSONParser() {
35 var jsonExpression = Token.New(JsonTokenType.BeginObject, JsonTokenType.BeginArray).Tag(0);
36 var jsonExpression = Token.New(JsonTokenType.BeginObject, JsonTokenType.BeginArray).Tag(0);
36
37
37 var valueExpression = Token.New(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
38 var valueExpression = Token.New(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
38 var memberExpression = Token.New(JsonTokenType.String).Cat(Token.New(JsonTokenType.NameSeparator)).Cat(valueExpression);
39 var memberExpression = Token.New(JsonTokenType.String).Cat(Token.New(JsonTokenType.NameSeparator)).Cat(valueExpression);
39 var objectExpression = memberExpression
40 var objectExpression = memberExpression
40 .Cat(
41 .Cat(
41 Token.New(JsonTokenType.ValueSeparator)
42 Token.New(JsonTokenType.ValueSeparator)
42 .Cat(memberExpression)
43 .Cat(memberExpression)
43 .EClosure()
44 .EClosure()
44 )
45 )
45 .Optional()
46 .Optional()
46 .Cat(Token.New(JsonTokenType.EndObject))
47 .Cat(Token.New(JsonTokenType.EndObject))
47 .Tag(0);
48 .Tag(0);
48 var arrayExpression = valueExpression
49 var arrayExpression = valueExpression
49 .Cat(
50 .Cat(
50 Token.New(JsonTokenType.ValueSeparator)
51 Token.New(JsonTokenType.ValueSeparator)
51 .Cat(valueExpression)
52 .Cat(valueExpression)
52 .EClosure()
53 .EClosure()
53 )
54 )
54 .Optional()
55 .Optional()
55 .Cat(Token.New(JsonTokenType.EndArray))
56 .Cat(Token.New(JsonTokenType.EndArray))
56 .Tag(0);
57 .Tag(0);
57
58
58 _jsonDFA = BuildDFA(jsonExpression).States;
59 _jsonDFA = BuildDFA(jsonExpression).States;
59 _objectDFA = BuildDFA(objectExpression).States;
60 _objectDFA = BuildDFA(objectExpression).States;
60 _arrayDFA = BuildDFA(arrayExpression).States;
61 _arrayDFA = BuildDFA(arrayExpression).States;
61 }
62 }
62
63
63 static EDFADefinition<JsonTokenType> BuildDFA(Token expr) {
64 static EDFADefinition<JsonTokenType> BuildDFA(Token expr) {
64 var builder = new DFABuilder();
65 var builder = new DFABuilder();
65 var dfa = new EDFADefinition<JsonTokenType>(_alphabet);
66 var dfa = new EDFADefinition<JsonTokenType>(_alphabet);
66 expr.Accept(builder);
67 expr.Accept(builder);
67
68
68 builder.BuildDFA(dfa);
69 builder.BuildDFA(dfa);
69 return dfa;
70 return dfa;
70 }
71 }
71
72
72 JSONScanner m_scanner;
73 JSONScanner m_scanner;
73 MemberContext m_memberContext;
74 MemberContext m_memberContext;
74
75
75 JSONElementType m_elementType;
76 JSONElementType m_elementType;
76 object m_elementValue;
77 object m_elementValue;
77
78
79 /// <summary>
80 /// Создает новый парсер на основе строки, содержащей JSON
81 /// </summary>
82 /// <param name="text"></param>
78 public JSONParser(string text)
83 public JSONParser(string text)
79 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty } ) {
84 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
80 Safe.ArgumentNotEmpty(text, "text");
85 Safe.ArgumentNotEmpty(text, "text");
81 m_scanner = new JSONScanner();
86 m_scanner = new JSONScanner();
82 m_scanner.Feed(text.ToCharArray());
87 m_scanner.Feed(text.ToCharArray());
83 }
88 }
84
89
90 /// <summary>
91 /// Создает новый экземпляр парсера, на основе текстового потока.
92 /// </summary>
93 /// <param name="reader">Текстовый поток.</param>
94 /// <param name="dispose">Признак того, что парсер должен конролировать время жизни входного потока.</param>
95 public JSONParser(TextReader reader, bool dispose)
96 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
97 Safe.ArgumentNotNull(reader, "reader");
98 m_scanner = new JSONScanner();
99 m_scanner.Feed(reader, dispose);
100 }
101
102 /// <summary>
103 /// Тип текущего элемента на котором стоит парсер.
104 /// </summary>
85 public JSONElementType ElementType {
105 public JSONElementType ElementType {
86 get { return m_elementType; }
106 get { return m_elementType; }
87 }
107 }
88
108
109 /// <summary>
110 /// Имя элемента - имя свойства родительского контейнера. Для элементов массивов и корневого всегда
111 /// пустая строка.
112 /// </summary>
89 public string ElementName {
113 public string ElementName {
90 get { return m_context.info.memberName; }
114 get { return m_context.info.memberName; }
91 }
115 }
92
116
117 /// <summary>
118 /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c>
119 /// </summary>
93 public object ElementValue {
120 public object ElementValue {
94 get { return m_elementValue; }
121 get { return m_elementValue; }
95 }
122 }
96
123
124 /// <summary>
125 /// Читает слеюудущий объект из потока
126 /// </summary>
127 /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
97 public bool Read() {
128 public bool Read() {
98 if (m_context.current == UNREACHEBLE_STATE)
129 if (m_context.current == UNREACHEBLE_STATE)
99 throw new InvalidOperationException("The parser is in invalid state");
130 throw new InvalidOperationException("The parser is in invalid state");
100 object tokenValue;
131 object tokenValue;
101 JsonTokenType tokenType;
132 JsonTokenType tokenType;
102 m_context.info.memberName = String.Empty;
133 m_context.info.memberName = String.Empty;
103 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
134 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
104 Move((int)tokenType);
135 Move((int)tokenType);
105 if (m_context.current == UNREACHEBLE_STATE)
136 if (m_context.current == UNREACHEBLE_STATE)
106 UnexpectedToken(tokenValue, tokenType);
137 UnexpectedToken(tokenValue, tokenType);
107 switch (tokenType) {
138 switch (tokenType) {
108 case JsonTokenType.BeginObject:
139 case JsonTokenType.BeginObject:
109 Switch(
140 Switch(
110 _objectDFA,
141 _objectDFA,
111 INITIAL_STATE,
142 INITIAL_STATE,
112 new JSONParserContext {
143 new JSONParserContext {
113 memberName = m_context.info.memberName,
144 memberName = m_context.info.memberName,
114 elementContext = JSONElementContext.Object
145 elementContext = JSONElementContext.Object
115 }
146 }
116 );
147 );
117 m_elementValue = null;
148 m_elementValue = null;
118 m_memberContext = MemberContext.MemberName;
149 m_memberContext = MemberContext.MemberName;
119 m_elementType = JSONElementType.BeginObject;
150 m_elementType = JSONElementType.BeginObject;
120 return true;
151 return true;
121 case JsonTokenType.EndObject:
152 case JsonTokenType.EndObject:
122 Restore();
153 Restore();
123 m_elementValue = null;
154 m_elementValue = null;
124 m_elementType = JSONElementType.EndObject;
155 m_elementType = JSONElementType.EndObject;
125 return true;
156 return true;
126 case JsonTokenType.BeginArray:
157 case JsonTokenType.BeginArray:
127 Switch(
158 Switch(
128 _arrayDFA,
159 _arrayDFA,
129 INITIAL_STATE,
160 INITIAL_STATE,
130 new JSONParserContext {
161 new JSONParserContext {
131 memberName = m_context.info.memberName,
162 memberName = m_context.info.memberName,
132 elementContext = JSONElementContext.Array
163 elementContext = JSONElementContext.Array
133 }
164 }
134 );
165 );
135 m_elementValue = null;
166 m_elementValue = null;
136 m_memberContext = MemberContext.MemberValue;
167 m_memberContext = MemberContext.MemberValue;
137 m_elementType = JSONElementType.BeginArray;
168 m_elementType = JSONElementType.BeginArray;
138 return true;
169 return true;
139 case JsonTokenType.EndArray:
170 case JsonTokenType.EndArray:
140 Restore();
171 Restore();
141 m_elementValue = null;
172 m_elementValue = null;
142 m_elementType = JSONElementType.EndArray;
173 m_elementType = JSONElementType.EndArray;
143 return true;
174 return true;
144 case JsonTokenType.String:
175 case JsonTokenType.String:
145 if (m_memberContext == MemberContext.MemberName) {
176 if (m_memberContext == MemberContext.MemberName) {
146 m_context.info.memberName = (string)tokenValue;
177 m_context.info.memberName = (string)tokenValue;
147 break;
178 break;
148 } else {
179 } else {
149 m_elementType = JSONElementType.Value;
180 m_elementType = JSONElementType.Value;
150 m_elementValue = tokenValue;
181 m_elementValue = tokenValue;
151 return true;
182 return true;
152 }
183 }
153 case JsonTokenType.Number:
184 case JsonTokenType.Number:
154 m_elementType = JSONElementType.Value;
185 m_elementType = JSONElementType.Value;
155 m_elementValue = tokenValue;
186 m_elementValue = tokenValue;
156 return true;
187 return true;
157 case JsonTokenType.Literal:
188 case JsonTokenType.Literal:
158 m_elementType = JSONElementType.Value;
189 m_elementType = JSONElementType.Value;
159 m_elementValue = ParseLiteral((string)tokenValue);
190 m_elementValue = ParseLiteral((string)tokenValue);
160 return true;
191 return true;
161 case JsonTokenType.NameSeparator:
192 case JsonTokenType.NameSeparator:
162 m_memberContext = MemberContext.MemberValue;
193 m_memberContext = MemberContext.MemberValue;
163 break;
194 break;
164 case JsonTokenType.ValueSeparator:
195 case JsonTokenType.ValueSeparator:
165 m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
196 m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
166 break;
197 break;
167 default:
198 default:
168 UnexpectedToken(tokenValue, tokenType);
199 UnexpectedToken(tokenValue, tokenType);
169 break;
200 break;
170 }
201 }
171 }
202 }
172 if (m_context.info.elementContext != JSONElementContext.None)
203 if (m_context.info.elementContext != JSONElementContext.None)
173 throw new ParserException("Unexpedted end of data");
204 throw new ParserException("Unexpedted end of data");
174 return false;
205 return false;
175 }
206 }
176
207
177 object ParseLiteral(string literal) {
208 object ParseLiteral(string literal) {
178 switch (literal) {
209 switch (literal) {
179 case "null":
210 case "null":
180 return null;
211 return null;
181 case "false" :
212 case "false":
182 return false;
213 return false;
183 case "true":
214 case "true":
184 return true;
215 return true;
185 default:
216 default:
186 UnexpectedToken(literal, JsonTokenType.Literal);
217 UnexpectedToken(literal, JsonTokenType.Literal);
187 return null; // avoid compliler error
218 return null; // avoid compliler error
188 }
219 }
189 }
220 }
190
221
191 void UnexpectedToken(object value, JsonTokenType tokenType) {
222 void UnexpectedToken(object value, JsonTokenType tokenType) {
192 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
223 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
193 }
224 }
194
225
195
226
227 /// <summary>
228 /// Признак конца потока
229 /// </summary>
196 public bool EOF {
230 public bool EOF {
197 get {
231 get {
198 return m_scanner.EOF;
232 return m_scanner.EOF;
199 }
233 }
200 }
234 }
235
236 protected virtual void Dispose(bool disposing) {
237 if (disposing) {
238 m_scanner.Dispose();
239 }
240 }
241
242 /// <summary>
243 /// Освобождает парсер и связанный с ним сканнер.
244 /// </summary>
245 public void Dispose() {
246 Dispose(true);
247 GC.SuppressFinalize(this);
248 }
249
250 ~JSONParser() {
251 Dispose(false);
252 }
201 }
253 }
202
254
203 }
255 }
@@ -1,278 +1,286
1 using Implab;
1 using Implab;
2 using Implab.JSON;
2 using Implab.JSON;
3 using Implab.Parsing;
3 using Implab.Parsing;
4 using System;
4 using System;
5 using System.Collections.Generic;
5 using System.Collections.Generic;
6 using System.Linq;
6 using System.Linq;
7 using System.Text;
7 using System.Text;
8 using System.Threading.Tasks;
8 using System.Threading.Tasks;
9 using System.Xml;
9 using System.Xml;
10
10
11 namespace ConsPlay {
11 namespace ConsPlay {
12 public class JSONXmlReader : XmlReader {
12 public class JSONXmlReader : XmlReader {
13
13
14 enum ValueContext {
14 enum ValueContext {
15 Undefined,
15 Undefined,
16 ElementStart,
16 ElementStart,
17 ElementValue,
17 ElementValue,
18 ElementEnd,
18 ElementEnd,
19 ElementEmpty
19 ElementEmpty
20 }
20 }
21
21
22 struct LocalNameContext {
22 struct LocalNameContext {
23 public string localName;
23 public string localName;
24 public bool isArray;
24 public bool isArray;
25 }
25 }
26
26
27 JSONParser m_parser;
27 JSONParser m_parser;
28 ValueContext m_valueContext;
28 ValueContext m_valueContext;
29 ReadState m_state = ReadState.Initial;
29 ReadState m_state = ReadState.Initial;
30 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
30 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
31 LocalNameContext m_localName;
31 LocalNameContext m_localName;
32 string m_rootName = "json";
32 string m_rootName = "json";
33 string m_prefix = String.Empty;
33 string m_prefix = String.Empty;
34 string m_namespaceUri = String.Empty;
34 string m_namespaceUri = String.Empty;
35 bool m_flattenArrays = false;
35 bool m_flattenArrays = false;
36 NameTable m_nameTable = new NameTable();
36 NameTable m_nameTable = new NameTable();
37 int m_depthCorrection = 0;
37 int m_depthCorrection = 0;
38
38
39 public JSONXmlReader(JSONParser parser) {
39 public JSONXmlReader(JSONParser parser) {
40 Safe.ArgumentNotNull(parser, "parser");
40 Safe.ArgumentNotNull(parser, "parser");
41 m_parser = parser;
41 m_parser = parser;
42 }
42 }
43
43
44 /// <summary>
44 /// <summary>
45 /// Always 0, JSON doesn't support attributes
45 /// Always 0, JSON doesn't support attributes
46 /// </summary>
46 /// </summary>
47 public override int AttributeCount {
47 public override int AttributeCount {
48 get { return 0; }
48 get { return 0; }
49 }
49 }
50
50
51 public override string BaseURI {
51 public override string BaseURI {
52 get { return String.Empty; }
52 get { return String.Empty; }
53 }
53 }
54
54
55 public override int Depth {
55 public override int Depth {
56 get {
56 get {
57 return m_localNameStack.Count+m_depthCorrection;
57 return m_localNameStack.Count+m_depthCorrection;
58 }
58 }
59 }
59 }
60
60
61 public override bool EOF {
61 public override bool EOF {
62 get { return m_parser.EOF; }
62 get { return m_parser.EOF; }
63 }
63 }
64
64
65 /// <summary>
65 /// <summary>
66 /// Always throws an exception
66 /// Always throws an exception
67 /// </summary>
67 /// </summary>
68 /// <param name="i"></param>
68 /// <param name="i"></param>
69 /// <returns></returns>
69 /// <returns></returns>
70 public override string GetAttribute(int i) {
70 public override string GetAttribute(int i) {
71 throw new ArgumentOutOfRangeException();
71 throw new ArgumentOutOfRangeException();
72 }
72 }
73
73
74 /// <summary>
74 /// <summary>
75 /// Always returns empty string
75 /// Always returns empty string
76 /// </summary>
76 /// </summary>
77 /// <param name="name"></param>
77 /// <param name="name"></param>
78 /// <param name="namespaceURI"></param>
78 /// <param name="namespaceURI"></param>
79 /// <returns></returns>
79 /// <returns></returns>
80 public override string GetAttribute(string name, string namespaceURI) {
80 public override string GetAttribute(string name, string namespaceURI) {
81 return String.Empty;
81 return String.Empty;
82 }
82 }
83
83
84 /// <summary>
84 /// <summary>
85 /// Always returns empty string
85 /// Always returns empty string
86 /// </summary>
86 /// </summary>
87 /// <param name="name"></param>
87 /// <param name="name"></param>
88 /// <returns></returns>
88 /// <returns></returns>
89 public override string GetAttribute(string name) {
89 public override string GetAttribute(string name) {
90 return String.Empty;
90 return String.Empty;
91 }
91 }
92
92
93 public override bool IsEmptyElement {
93 public override bool IsEmptyElement {
94 get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
94 get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
95 }
95 }
96
96
97 public override string LocalName {
97 public override string LocalName {
98 get { return m_localName.localName; }
98 get { return m_localName.localName; }
99 }
99 }
100
100
101 public override string LookupNamespace(string prefix) {
101 public override string LookupNamespace(string prefix) {
102 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
102 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
103 return m_namespaceUri;
103 return m_namespaceUri;
104 else
104 else
105 return String.Empty;
105 return String.Empty;
106 }
106 }
107
107
108 public override bool MoveToAttribute(string name, string ns) {
108 public override bool MoveToAttribute(string name, string ns) {
109 return false;
109 return false;
110 }
110 }
111
111
112 public override bool MoveToAttribute(string name) {
112 public override bool MoveToAttribute(string name) {
113 return false;
113 return false;
114 }
114 }
115
115
116 public override bool MoveToElement() {
116 public override bool MoveToElement() {
117 return false;
117 return false;
118 }
118 }
119
119
120 public override bool MoveToFirstAttribute() {
120 public override bool MoveToFirstAttribute() {
121 return false;
121 return false;
122 }
122 }
123
123
124 public override bool MoveToNextAttribute() {
124 public override bool MoveToNextAttribute() {
125 return false;
125 return false;
126 }
126 }
127
127
128 public override XmlNameTable NameTable {
128 public override XmlNameTable NameTable {
129 get { return m_nameTable; }
129 get { return m_nameTable; }
130 }
130 }
131
131
132 public override string NamespaceURI {
132 public override string NamespaceURI {
133 get { return m_namespaceUri; }
133 get { return m_namespaceUri; }
134 }
134 }
135
135
136 public override XmlNodeType NodeType {
136 public override XmlNodeType NodeType {
137 get {
137 get {
138 switch (m_parser.ElementType) {
138 switch (m_parser.ElementType) {
139 case JSONElementType.BeginObject:
139 case JSONElementType.BeginObject:
140 case JSONElementType.BeginArray:
140 case JSONElementType.BeginArray:
141 return XmlNodeType.Element;
141 return XmlNodeType.Element;
142 case JSONElementType.EndObject:
142 case JSONElementType.EndObject:
143 case JSONElementType.EndArray:
143 case JSONElementType.EndArray:
144 return XmlNodeType.EndElement;
144 return XmlNodeType.EndElement;
145 case JSONElementType.Value:
145 case JSONElementType.Value:
146 switch (m_valueContext) {
146 switch (m_valueContext) {
147 case ValueContext.ElementStart:
147 case ValueContext.ElementStart:
148 case ValueContext.ElementEmpty:
148 case ValueContext.ElementEmpty:
149 return XmlNodeType.Element;
149 return XmlNodeType.Element;
150 case ValueContext.ElementValue:
150 case ValueContext.ElementValue:
151 return XmlNodeType.Text;
151 return XmlNodeType.Text;
152 case ValueContext.ElementEnd:
152 case ValueContext.ElementEnd:
153 return XmlNodeType.EndElement;
153 return XmlNodeType.EndElement;
154 default:
154 default:
155 throw new InvalidOperationException();
155 throw new InvalidOperationException();
156 }
156 }
157 default:
157 default:
158 throw new InvalidOperationException();
158 throw new InvalidOperationException();
159 }
159 }
160 }
160 }
161 }
161 }
162
162
163 public override string Prefix {
163 public override string Prefix {
164 get { return m_prefix; }
164 get { return m_prefix; }
165 }
165 }
166
166
167 public override bool Read() {
167 public override bool Read() {
168 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
168 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
169 return false;
169 return false;
170
170
171 if (m_state == ReadState.Initial)
171 if (m_state == ReadState.Initial)
172 m_state = System.Xml.ReadState.Interactive;
172 m_state = System.Xml.ReadState.Interactive;
173
173
174 try {
174 try {
175 switch (m_parser.ElementType) {
175 switch (m_parser.ElementType) {
176 case JSONElementType.Value:
176 case JSONElementType.Value:
177 switch (m_valueContext) {
177 switch (m_valueContext) {
178 case ValueContext.ElementStart:
178 case ValueContext.ElementStart:
179 SetLocalName(String.Empty);
179 SetLocalName(String.Empty);
180 m_valueContext = ValueContext.ElementValue;
180 m_valueContext = ValueContext.ElementValue;
181 return true;
181 return true;
182 case ValueContext.ElementValue:
182 case ValueContext.ElementValue:
183 RestoreLocalName();
183 RestoreLocalName();
184 m_valueContext = ValueContext.ElementEnd;
184 m_valueContext = ValueContext.ElementEnd;
185 return true;
185 return true;
186 case ValueContext.ElementEmpty:
186 case ValueContext.ElementEmpty:
187 case ValueContext.ElementEnd:
187 case ValueContext.ElementEnd:
188 RestoreLocalName();
188 RestoreLocalName();
189 break;
189 break;
190 }
190 }
191 break;
191 break;
192 case JSONElementType.EndArray:
192 case JSONElementType.EndArray:
193 case JSONElementType.EndObject:
193 case JSONElementType.EndObject:
194 RestoreLocalName();
194 RestoreLocalName();
195 break;
195 break;
196 }
196 }
197 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
197 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
198 while (m_parser.Read()) {
198 while (m_parser.Read()) {
199 if (!String.IsNullOrEmpty(m_parser.ElementName))
199 if (!String.IsNullOrEmpty(m_parser.ElementName))
200 itemName = m_parser.ElementName;
200 itemName = m_parser.ElementName;
201
201
202 switch (m_parser.ElementType) {
202 switch (m_parser.ElementType) {
203 case JSONElementType.BeginArray:
203 case JSONElementType.BeginArray:
204 if (m_flattenArrays && !m_localName.isArray) {
204 if (m_flattenArrays && !m_localName.isArray) {
205 m_depthCorrection--;
205 m_depthCorrection--;
206 SetLocalName(itemName, true);
206 SetLocalName(itemName, true);
207 continue;
207 continue;
208 } else {
208 } else {
209 SetLocalName(itemName, true);
209 SetLocalName(itemName, true);
210 }
210 }
211 break;
211 break;
212 case JSONElementType.BeginObject:
212 case JSONElementType.BeginObject:
213 SetLocalName(itemName);
213 SetLocalName(itemName);
214 break;
214 break;
215 case JSONElementType.EndArray:
215 case JSONElementType.EndArray:
216 if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
216 if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
217 RestoreLocalName();
217 RestoreLocalName();
218 m_depthCorrection++;
218 m_depthCorrection++;
219 continue;
219 continue;
220 }
220 }
221 break;
221 break;
222 case JSONElementType.EndObject:
222 case JSONElementType.EndObject:
223 break;
223 break;
224 case JSONElementType.Value:
224 case JSONElementType.Value:
225 SetLocalName(itemName);
225 SetLocalName(itemName);
226 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
226 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
227 break;
227 break;
228 default:
228 default:
229 break;
229 break;
230 }
230 }
231 return true;
231 return true;
232 }
232 }
233
233
234 m_state = System.Xml.ReadState.EndOfFile;
234 m_state = System.Xml.ReadState.EndOfFile;
235 return false;
235 return false;
236 } catch {
236 } catch {
237 m_state = System.Xml.ReadState.Error;
237 m_state = System.Xml.ReadState.Error;
238 throw;
238 throw;
239 }
239 }
240 }
240 }
241
241
242 public override bool ReadAttributeValue() {
242 public override bool ReadAttributeValue() {
243 return false;
243 return false;
244 }
244 }
245
245
246 public override ReadState ReadState {
246 public override ReadState ReadState {
247 get { return m_state; }
247 get { return m_state; }
248 }
248 }
249
249
250 public override void ResolveEntity() {
250 public override void ResolveEntity() {
251 // do nothing
251 // do nothing
252 }
252 }
253
253
254 public override string Value {
254 public override string Value {
255 get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
255 get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
256 }
256 }
257
257
258 void SetLocalName(string name) {
258 void SetLocalName(string name) {
259 m_localNameStack.Push(m_localName);
259 m_localNameStack.Push(m_localName);
260 m_localName.localName = name;
260 m_localName.localName = name;
261 m_localName.isArray = false;
261 m_localName.isArray = false;
262 }
262 }
263
263
264 void SetLocalName(string name, bool isArray) {
264 void SetLocalName(string name, bool isArray) {
265 m_localNameStack.Push(m_localName);
265 m_localNameStack.Push(m_localName);
266 m_localName.localName = name;
266 m_localName.localName = name;
267 m_localName.isArray = isArray;
267 m_localName.isArray = isArray;
268 }
268 }
269
269
270 void RestoreLocalName() {
270 void RestoreLocalName() {
271 m_localName = m_localNameStack.Pop();
271 m_localName = m_localNameStack.Pop();
272 }
272 }
273
273
274 public override void Close() {
274 public override void Close() {
275 m_state = System.Xml.ReadState.EndOfFile ;
275
276 }
276 }
277
278 protected override void Dispose(bool disposing) {
279 if (disposing) {
280 m_parser.Dispose();
281 }
282 base.Dispose(disposing);
283 }
284
277 }
285 }
278 }
286 }
@@ -1,23 +1,23
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5 using System.Text;
5 using System.Text;
6 using System.Threading.Tasks;
6 using System.Threading.Tasks;
7
7
8 namespace Implab.Parsing {
8 namespace Implab.Parsing {
9 public class Alphabet: AlphabetBase<char> {
9 public class Alphabet: AlphabetBase<char> {
10
10
11 public Alphabet()
12 : base(char.MaxValue + 1) {
13 }
14
11 public override int GetSymbolIndex(char symbol) {
15 public override int GetSymbolIndex(char symbol) {
12 return symbol;
16 return symbol;
13 }
17 }
14
18
15 public override IEnumerable<char> InputSymbols {
19 public override IEnumerable<char> InputSymbols {
16 get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); }
20 get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); }
17 }
21 }
18
19 protected override int MapSize {
20 get { return char.MaxValue + 1; }
21 }
22 }
22 }
23 }
23 }
@@ -1,103 +1,100
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Diagnostics;
4 using System.Diagnostics;
5 using System.Linq;
5 using System.Linq;
6 using System.Text;
6 using System.Text;
7 using System.Threading.Tasks;
7 using System.Threading.Tasks;
8
8
9 namespace Implab.Parsing {
9 namespace Implab.Parsing {
10 public abstract class AlphabetBase<T> : IAlphabet<T> {
10 public abstract class AlphabetBase<T> : IAlphabet<T> {
11 public const int UNCLASSIFIED = 0;
11 public const int UNCLASSIFIED = 0;
12
12
13 int m_nextId = 1;
13 int m_nextId = 1;
14 int[] m_map;
14 int[] m_map;
15
15
16 public int Count {
16 public int Count {
17 get { return m_nextId; }
17 get { return m_nextId; }
18 }
18 }
19
19
20 protected AlphabetBase() {
20 protected AlphabetBase(int mapSize) {
21 m_map = new int[MapSize];
21 m_map = new int[mapSize];
22 }
22 }
23
23
24 protected AlphabetBase(int[] map) {
24 protected AlphabetBase(int[] map) {
25 Debug.Assert(map != null);
25 Debug.Assert(map != null);
26 Debug.Assert(map.Length == MapSize);
27
26
28 m_map = map;
27 m_map = map;
29 m_nextId = map.Max() + 1;
28 m_nextId = map.Max() + 1;
30 }
29 }
31
30
32 public int DefineSymbol(T symbol) {
31 public int DefineSymbol(T symbol) {
33 var index = GetSymbolIndex(symbol);
32 var index = GetSymbolIndex(symbol);
34 if (m_map[index] == UNCLASSIFIED)
33 if (m_map[index] == UNCLASSIFIED)
35 m_map[index] = m_nextId++;
34 m_map[index] = m_nextId++;
36 return m_map[index];
35 return m_map[index];
37 }
36 }
38
37
39 public int DefineClass(IEnumerable<T> symbols) {
38 public int DefineClass(IEnumerable<T> symbols) {
40 Safe.ArgumentNotNull(symbols, "symbols");
39 Safe.ArgumentNotNull(symbols, "symbols");
41 symbols = symbols.Distinct();
40 symbols = symbols.Distinct();
42
41
43 foreach (var symbol in symbols) {
42 foreach (var symbol in symbols) {
44 var index = GetSymbolIndex(symbol);
43 var index = GetSymbolIndex(symbol);
45 if (m_map[index] == UNCLASSIFIED)
44 if (m_map[index] == UNCLASSIFIED)
46 m_map[GetSymbolIndex(symbol)] = m_nextId;
45 m_map[GetSymbolIndex(symbol)] = m_nextId;
47 else
46 else
48 throw new InvalidOperationException(String.Format("Symbol '{0}' already in use", symbol));
47 throw new InvalidOperationException(String.Format("Symbol '{0}' already in use", symbol));
49 }
48 }
50 return m_nextId++;
49 return m_nextId++;
51 }
50 }
52
51
53 public List<T>[] CreateReverseMap() {
52 public List<T>[] CreateReverseMap() {
54 return
53 return
55 Enumerable.Range(UNCLASSIFIED, Count)
54 Enumerable.Range(UNCLASSIFIED, Count)
56 .Select(
55 .Select(
57 i => InputSymbols
56 i => InputSymbols
58 .Where(x => i != UNCLASSIFIED && m_map[GetSymbolIndex(x)] == i)
57 .Where(x => i != UNCLASSIFIED && m_map[GetSymbolIndex(x)] == i)
59 .ToList()
58 .ToList()
60 )
59 )
61 .ToArray();
60 .ToArray();
62 }
61 }
63
62
64 public int[] Reclassify(IAlphabet<T> newAlphabet, IEnumerable<ICollection<int>> classes) {
63 public int[] Reclassify(IAlphabet<T> newAlphabet, IEnumerable<ICollection<int>> classes) {
65 Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
64 Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
66 Safe.ArgumentNotNull(classes, "classes");
65 Safe.ArgumentNotNull(classes, "classes");
67 var reverseMap = CreateReverseMap();
66 var reverseMap = CreateReverseMap();
68
67
69 int[] translationMap = new int[Count];
68 int[] translationMap = new int[Count];
70
69
71 foreach (var scl in classes) {
70 foreach (var scl in classes) {
72 // skip if the supper class contains the unclassified element
71 // skip if the supper class contains the unclassified element
73 if (scl.Contains(UNCLASSIFIED))
72 if (scl.Contains(UNCLASSIFIED))
74 continue;
73 continue;
75 var range = new List<T>();
74 var range = new List<T>();
76 foreach (var cl in scl) {
75 foreach (var cl in scl) {
77 if (cl < 0 || cl >= reverseMap.Length)
76 if (cl < 0 || cl >= reverseMap.Length)
78 throw new ArgumentOutOfRangeException(String.Format("Class {0} is not valid for the current alphabet", cl));
77 throw new ArgumentOutOfRangeException(String.Format("Class {0} is not valid for the current alphabet", cl));
79 range.AddRange(reverseMap[cl]);
78 range.AddRange(reverseMap[cl]);
80 }
79 }
81 var newClass = newAlphabet.DefineClass(range);
80 var newClass = newAlphabet.DefineClass(range);
82 foreach (var cl in scl)
81 foreach (var cl in scl)
83 translationMap[cl] = newClass;
82 translationMap[cl] = newClass;
84 }
83 }
85
84
86 return translationMap;
85 return translationMap;
87 }
86 }
88
87
89 public int Translate(T symbol) {
88 public int Translate(T symbol) {
90 return m_map[GetSymbolIndex(symbol)];
89 return m_map[GetSymbolIndex(symbol)];
91 }
90 }
92
91
93 public abstract int GetSymbolIndex(T symbol);
92 public abstract int GetSymbolIndex(T symbol);
94
93
95 public abstract IEnumerable<T> InputSymbols { get; }
94 public abstract IEnumerable<T> InputSymbols { get; }
96
95
97 protected abstract int MapSize { get; }
98
99 public int[] GetTranslationMap() {
96 public int[] GetTranslationMap() {
100 return m_map;
97 return m_map;
101 }
98 }
102 }
99 }
103 }
100 }
@@ -1,68 +1,67
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Diagnostics;
4 using System.Globalization;
5 using System.Globalization;
5 using System.Linq;
6 using System.Linq;
6 using System.Text;
7 using System.Text;
7 using System.Threading.Tasks;
8 using System.Threading.Tasks;
8
9
9 namespace Implab.Parsing {
10 namespace Implab.Parsing {
10 /// <summary>
11 /// <summary>
11 /// Алфавит символами которого являются элементы перечислений.
12 /// Алфавит символами которого являются элементы перечислений.
12 /// </summary>
13 /// </summary>
13 /// <typeparam name="T">Тип перечислений</typeparam>
14 /// <typeparam name="T">Тип перечислений</typeparam>
14 public class EnumAlphabet<T> : AlphabetBase<T> where T : struct, IConvertible {
15 public class EnumAlphabet<T> : AlphabetBase<T> where T : struct, IConvertible {
15 static readonly T[] _symbols;
16 static readonly T[] _symbols;
16 static readonly EnumAlphabet<T> _fullAlphabet;
17 static readonly EnumAlphabet<T> _fullAlphabet;
17
18
18 static EnumAlphabet() {
19 static EnumAlphabet() {
19 if (!typeof(T).IsEnum)
20 if (!typeof(T).IsEnum)
20 throw new InvalidOperationException("Invalid generic parameter, enumeration is required");
21 throw new InvalidOperationException("Invalid generic parameter, enumeration is required");
21
22
22 if (Enum.GetUnderlyingType(typeof(T)) != typeof(Int32))
23 if (Enum.GetUnderlyingType(typeof(T)) != typeof(Int32))
23 throw new InvalidOperationException("Only enums based on Int32 are supported");
24 throw new InvalidOperationException("Only enums based on Int32 are supported");
24
25
25 _symbols = ((T[])Enum.GetValues(typeof(T)))
26 _symbols = ((T[])Enum.GetValues(typeof(T)))
26 .OrderBy(x => x.ToInt32(CultureInfo.InvariantCulture))
27 .OrderBy(x => x.ToInt32(CultureInfo.InvariantCulture))
27 .ToArray();
28 .ToArray();
28
29
29 if (
30 if (
30 _symbols[_symbols.Length - 1].ToInt32(CultureInfo.InvariantCulture) >= _symbols.Length
31 _symbols[_symbols.Length - 1].ToInt32(CultureInfo.InvariantCulture) >= _symbols.Length
31 || _symbols[0].ToInt32(CultureInfo.InvariantCulture) != 0
32 || _symbols[0].ToInt32(CultureInfo.InvariantCulture) != 0
32 )
33 )
33 throw new InvalidOperationException("The specified enumeration must be zero-based and continuously numbered");
34 throw new InvalidOperationException("The specified enumeration must be zero-based and continuously numbered");
34
35
35 _fullAlphabet = new EnumAlphabet<T>(_symbols.Select(x => x.ToInt32(CultureInfo.InvariantCulture)).ToArray());
36 _fullAlphabet = new EnumAlphabet<T>(_symbols.Select(x => x.ToInt32(CultureInfo.InvariantCulture)).ToArray());
36 }
37 }
37
38
38
39
39
40
40 public static EnumAlphabet<T> FullAlphabet {
41 public static EnumAlphabet<T> FullAlphabet {
41 get {
42 get {
42 return _fullAlphabet;
43 return _fullAlphabet;
43 }
44 }
44 }
45 }
45
46
46
47
47 public EnumAlphabet()
48 public EnumAlphabet()
48 : base() {
49 : base(_symbols.Length) {
49 }
50 }
50
51
51 public EnumAlphabet(int[] map)
52 public EnumAlphabet(int[] map)
52 : base(map) {
53 : base(map) {
54 Debug.Assert(map.Length == _symbols.Length);
53 }
55 }
54
56
55
57
56 public override int GetSymbolIndex(T symbol) {
58 public override int GetSymbolIndex(T symbol) {
57 return symbol.ToInt32(CultureInfo.InvariantCulture);
59 return symbol.ToInt32(CultureInfo.InvariantCulture);
58 }
60 }
59
61
60 public override IEnumerable<T> InputSymbols {
62 public override IEnumerable<T> InputSymbols {
61 get { return _symbols; }
63 get { return _symbols; }
62 }
64 }
63
65
64 protected override int MapSize {
65 get { return _symbols.Length; }
66 }
67 }
66 }
68 }
67 }
@@ -1,216 +1,259
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.IO;
4 using System.Linq;
5 using System.Linq;
5 using System.Text;
6 using System.Text;
6 using System.Threading.Tasks;
7 using System.Threading.Tasks;
7
8
8 namespace Implab.Parsing {
9 namespace Implab.Parsing {
9 /// <summary>
10 /// <summary>
10 /// Базовый класс для разбора потока входных символов на токены.
11 /// Базовый класс для разбора потока входных символов на токены.
11 /// </summary>
12 /// </summary>
12 /// <remarks>
13 /// <remarks>
13 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
14 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
14 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
15 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
15 /// конца токена и допустимости текущего символа.
16 /// конца токена и допустимости текущего символа.
16 /// </remarks>
17 /// </remarks>
17 public class Scanner {
18 public abstract class Scanner : Disposable {
18 struct ScannerConfig {
19 struct ScannerConfig {
19 public DFAStateDescriptior[] states;
20 public DFAStateDescriptior[] states;
20 public int[] alphabetMap;
21 public int[] alphabetMap;
21 }
22 }
22
23
23 Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
24 Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
24
25
25 DFAStateDescriptior[] m_states;
26 DFAStateDescriptior[] m_states;
26 int[] m_alphabetMap;
27 int[] m_alphabetMap;
27
28
28 protected DFAStateDescriptior m_currentState;
29 protected DFAStateDescriptior m_currentState;
29 int m_previewCode;
30 int m_previewCode;
30
31
31 protected int m_tokenLen = 0;
32 protected int m_tokenLen = 0;
32 protected int m_tokenOffset;
33 protected int m_tokenOffset;
33
34
34 protected char[] m_buffer;
35 protected char[] m_buffer;
35 protected int m_bufferSize;
36 protected int m_bufferSize;
36 protected int m_pointer;
37 protected int m_pointer;
37
38
38 public Scanner(CDFADefinition definition, string text) {
39 TextReader m_reader;
39 Safe.ArgumentNotNull(definition, "definition");
40 bool m_disposeReader;
40 Safe.ArgumentNotEmpty(text, "text");
41 int m_chunkSize = 1024; // 1k
41
42 int m_limit = 10 * 1024 * 1024; // 10Mb
42 m_states = definition.States;
43 m_alphabetMap = definition.Alphabet.GetTranslationMap();
44
45 Feed(text.ToCharArray());
46 }
47
43
48 public Scanner(CDFADefinition definition) {
44 public Scanner(CDFADefinition definition) {
49 Safe.ArgumentNotNull(definition, "definition");
45 Safe.ArgumentNotNull(definition, "definition");
50
46
51 m_states = definition.States;
47 m_states = definition.States;
52 m_alphabetMap = definition.Alphabet.GetTranslationMap();
48 m_alphabetMap = definition.Alphabet.GetTranslationMap();
53
49
54 Feed(new char[0]);
50 Feed(new char[0]);
55 }
51 }
56
52
57 /// <summary>
53 /// <summary>
58 /// Заполняет входными данными буффер.
54 /// Заполняет входными данными буффер.
59 /// </summary>
55 /// </summary>
60 /// <param name="data">Данные для обработки.</param>
56 /// <param name="data">Данные для обработки.</param>
61 /// <remarks>Копирование данных не происходит, переданный массив используется в
57 /// <remarks>Копирование данных не происходит, переданный массив используется в
62 /// качестве входного буффера.</remarks>
58 /// качестве входного буффера.</remarks>
63 public void Feed(char[] data) {
59 public void Feed(char[] data) {
64 Safe.ArgumentNotNull(data, "data");
60 Safe.ArgumentNotNull(data, "data");
65
61
66 Feed(data, data.Length);
62 Feed(data, data.Length);
67 }
63 }
68
64
69 /// <summary>
65 /// <summary>
70 /// Заполняет буффур чтения входными данными.
66 /// Заполняет буффур чтения входными данными.
71 /// </summary>
67 /// </summary>
72 /// <param name="data">Данные для обработки.</param>
68 /// <param name="data">Данные для обработки.</param>
73 /// <param name="length">Длина данных для обработки.</param>
69 /// <param name="length">Длина данных для обработки.</param>
74 /// <remarks>Копирование данных не происходит, переданный массив используется в
70 /// <remarks>Копирование данных не происходит, переданный массив используется в
75 /// качестве входного буффера.</remarks>
71 /// качестве входного буффера.</remarks>
76 public void Feed(char[] data, int length) {
72 public void Feed(char[] data, int length) {
77 Safe.ArgumentNotNull(data, "data");
73 Safe.ArgumentNotNull(data, "data");
78 Safe.ArgumentInRange(length, 0, data.Length, "length");
74 Safe.ArgumentInRange(length, 0, data.Length, "length");
75 AssertNotDisposed();
79
76
80 m_pointer = -1;
77 m_pointer = -1;
81 m_buffer = data;
78 m_buffer = data;
82 m_bufferSize = length;
79 m_bufferSize = length;
83 Shift();
80 Shift();
84 }
81 }
85
82
83 public void Feed(TextReader reader, bool dispose) {
84 Safe.ArgumentNotNull(reader, "reader");
85 AssertNotDisposed();
86
87 if (m_reader != null && m_disposeReader)
88 m_reader.Dispose();
89
90 m_reader = reader;
91 m_disposeReader = dispose;
92 m_pointer = -1;
93 m_buffer = new char[m_chunkSize];
94 m_bufferSize = 0;
95 Shift();
96 }
97
86 /// <summary>
98 /// <summary>
87 /// Получает текущий токен в виде строки.
99 /// Получает текущий токен в виде строки.
88 /// </summary>
100 /// </summary>
89 /// <returns></returns>
101 /// <returns></returns>
90 public string GetTokenValue() {
102 protected string GetTokenValue() {
91 return new String(m_buffer, m_tokenOffset, m_tokenLen);
103 return new String(m_buffer, m_tokenOffset, m_tokenLen);
92 }
104 }
93
105
94 /// <summary>
106 /// <summary>
95 /// Метки текущего токена, которые были назначены в регулярном выражении.
107 /// Метки текущего токена, которые были назначены в регулярном выражении.
96 /// </summary>
108 /// </summary>
97 public int[] TokenTags {
109 protected int[] TokenTags {
98 get {
110 get {
99 return m_currentState.tag;
111 return m_currentState.tag;
100 }
112 }
101 }
113 }
102
114
103 /// <summary>
115 /// <summary>
104 /// Признак конца данных
116 /// Признак конца данных
105 /// </summary>
117 /// </summary>
106 public bool EOF {
118 public bool EOF {
107 get {
119 get {
108 return m_pointer >= m_bufferSize;
120 return m_pointer >= m_bufferSize;
109 }
121 }
110 }
122 }
111
123
112 /// <summary>
124 /// <summary>
113 /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
125 /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
114 /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
126 /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
115 /// котором находится токен.
127 /// котором находится токен.
116 /// </summary>
128 /// </summary>
117 /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
129 /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
118 protected bool ReadTokenInternal() {
130 protected bool ReadTokenInternal() {
119 if (m_pointer >= m_bufferSize)
131 if (m_pointer >= m_bufferSize)
120 return false;
132 return false;
121
133
122 m_currentState = m_states[CDFADefinition.INITIAL_STATE];
134 m_currentState = m_states[CDFADefinition.INITIAL_STATE];
123 m_tokenLen = 0;
135 m_tokenLen = 0;
124 m_tokenOffset = m_pointer;
136 m_tokenOffset = m_pointer;
125 int nextState = CDFADefinition.UNREACHEBLE_STATE;
137 int nextState = CDFADefinition.UNREACHEBLE_STATE;
126 do {
138 do {
127 nextState = m_currentState.transitions[m_previewCode];
139 nextState = m_currentState.transitions[m_previewCode];
128 if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
140 if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
129 if (m_currentState.final)
141 if (m_currentState.final)
130 return true;
142 return true;
131 else
143 else
132 throw new ParserException(
144 throw new ParserException(
133 String.Format(
145 String.Format(
134 "Unexpected symbol '{0}', at pos {1}",
146 "Unexpected symbol '{0}', at pos {1}",
135 m_buffer[m_pointer],
147 m_buffer[m_pointer],
136 Position
148 Position
137 )
149 )
138 );
150 );
139 } else {
151 } else {
140 m_currentState = m_states[nextState];
152 m_currentState = m_states[nextState];
141 m_tokenLen++;
153 m_tokenLen++;
142 }
154 }
143
155
144 } while (Shift());
156 } while (Shift());
145
157
146 // END OF DATA
158 // END OF DATA
147 if (!m_currentState.final)
159 if (!m_currentState.final)
148 throw new ParserException("Unexpected end of data");
160 throw new ParserException("Unexpected end of data");
149
161
150 return true;
162 return true;
151 }
163 }
152
164
153
165
154 bool Shift() {
166 bool Shift() {
155 m_pointer++;
167 m_pointer++;
156
168
157 if (m_pointer >= m_bufferSize) {
169 if (m_pointer >= m_bufferSize) {
158 return ReadNextChunk();
170 return ReadNextChunk();
159 }
171 }
160
172
161 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
173 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
162
174
163 return true;
175 return true;
164 }
176 }
165
177
166 /// <summary>
178 bool ReadNextChunk() {
167 /// Вызывается по достижению конца входного буффера для получения
179 if (m_reader == null)
168 /// новых данных.
180 return false;
169 /// </summary>
181
170 /// <returns><c>true</c> - новые двнные получены, можно продолжать обработку.</returns>
182 // extend buffer if nesessary
171 protected virtual bool ReadNextChunk() {
183 if (m_pointer + m_chunkSize > m_buffer.Length) {
172 return false;
184 // trim unused buffer head
185 var size = m_tokenLen + m_chunkSize;
186 if (size >= m_limit)
187 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
188 var temp = new char[size];
189 Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
190 m_pointer -= m_tokenOffset;
191 m_bufferSize -= m_tokenOffset;
192 m_tokenOffset = 0;
193 m_buffer = temp;
194 }
195
196 var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
197 if (read == 0)
198 return false;
199
200 m_bufferSize += read;
201
202 return true;
173 }
203 }
174
204
175 /// <summary>
205 /// <summary>
176 /// Позиция сканнера во входном буфере
206 /// Позиция сканнера во входном буфере
177 /// </summary>
207 /// </summary>
178 public int Position {
208 public int Position {
179 get {
209 get {
180 return m_pointer + 1;
210 return m_pointer + 1;
181 }
211 }
182 }
212 }
183
213
184 /// <summary>
214 /// <summary>
185 /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
215 /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
186 /// группировки.
216 /// группировки.
187 /// </summary>
217 /// </summary>
188 /// <param name="states">Таблица состояний нового ДКА</param>
218 /// <param name="states">Таблица состояний нового ДКА</param>
189 /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
219 /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
190 protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
220 protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
191 Safe.ArgumentNotNull(states, "dfa");
221 Safe.ArgumentNotNull(states, "dfa");
192
222
193 m_defs.Push(new ScannerConfig {
223 m_defs.Push(new ScannerConfig {
194 states = m_states,
224 states = m_states,
195 alphabetMap = m_alphabetMap
225 alphabetMap = m_alphabetMap
196 });
226 });
197
227
198 m_states = states;
228 m_states = states;
199 m_alphabetMap = alphabet;
229 m_alphabetMap = alphabet;
200
230
201 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
231 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
202 }
232 }
203
233
204 /// <summary>
234 /// <summary>
205 /// Восстанавливает предыдущей ДКА сканнера.
235 /// Восстанавливает предыдущей ДКА сканнера.
206 /// </summary>
236 /// </summary>
207 protected void Restore() {
237 protected void Restore() {
208 if (m_defs.Count == 0)
238 if (m_defs.Count == 0)
209 throw new InvalidOperationException();
239 throw new InvalidOperationException();
210 var prev = m_defs.Pop();
240 var prev = m_defs.Pop();
211 m_states = prev.states;
241 m_states = prev.states;
212 m_alphabetMap = prev.alphabetMap;
242 m_alphabetMap = prev.alphabetMap;
213 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
243 m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
214 }
244 }
245
246 protected override void Dispose(bool disposing) {
247 if (disposing) {
248 if (m_reader != null && m_disposeReader)
249 m_reader.Dispose();
250 m_buffer = null;
251 m_bufferSize = 0;
252 m_pointer = 0;
253 m_tokenLen = 0;
254 m_tokenOffset = 0;
255 }
256 base.Dispose(disposing);
257 }
215 }
258 }
216 }
259 }
@@ -1,25 +1,27
1 using System.Reflection;
1 using System.Reflection;
2 using System.Runtime.CompilerServices;
2 using System.Runtime.CompilerServices;
3 using System.Runtime.InteropServices;
3
4
4 // Information about this assembly is defined by the following attributes.
5 // Information about this assembly is defined by the following attributes.
5 // Change them to the values specific to your project.
6 // Change them to the values specific to your project.
6
7
7 [assembly: AssemblyTitle("Implab")]
8 [assembly: AssemblyTitle("Implab")]
8 [assembly: AssemblyDescription("Tools")]
9 [assembly: AssemblyDescription("Tools")]
9 [assembly: AssemblyConfiguration("")]
10 [assembly: AssemblyConfiguration("")]
10 [assembly: AssemblyCompany("")]
11 [assembly: AssemblyCompany("")]
11 [assembly: AssemblyProduct("")]
12 [assembly: AssemblyProduct("")]
12 [assembly: AssemblyCopyright("Implab")]
13 [assembly: AssemblyCopyright("Implab")]
13 [assembly: AssemblyTrademark("")]
14 [assembly: AssemblyTrademark("")]
14 // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
15 // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
15 // The form "{Major}.{Minor}.*" will automatically update the build and revision,
16 // The form "{Major}.{Minor}.*" will automatically update the build and revision,
16 // and "{Major}.{Minor}.{Build}.*" will update just the revision.
17 // and "{Major}.{Minor}.{Build}.*" will update just the revision.
17
18
18 [assembly: AssemblyVersion("1.0.*")]
19 [assembly: AssemblyVersion("1.0.*")]
20 [assembly: ComVisible(false)]
19
21
20 // The following attributes are used to specify the signing key for the assembly,
22 // The following attributes are used to specify the signing key for the assembly,
21 // if desired. See the Mono documentation for more information about signing.
23 // if desired. See the Mono documentation for more information about signing.
22
24
23 //[assembly: AssemblyDelaySign(false)]
25 //[assembly: AssemblyDelaySign(false)]
24 //[assembly: AssemblyKeyFile("")]
26 //[assembly: AssemblyKeyFile("")]
25
27
General Comments 0
You need to be logged in to leave comments. Login now