##// END OF EJS Templates
implab » ImplabNet
RSS
JSON moved to Formats namespace...
cin -
r163:419aa51b04fd ref20160224
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,46
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4
5 namespace Implab.Automaton {
6 public class DummyAlphabet : IAlphabet<int> {
7 readonly int m_size;
8 public DummyAlphabet(int size) {
9 Safe.ArgumentAssert(size > 0);
10 m_size = 0;
11 }
12
13 #region IAlphabet implementation
14
15 public List<int>[] CreateReverseMap() {
16 Enumerable.Range(0, m_size).ToArray();
17 }
18
19 public int[] Reclassify(IAlphabetBuilder<int> newAlphabet, IEnumerable<IEnumerable<int>> classes) {
20 Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
21 Safe.ArgumentNotNull(classes, "classes");
22 var map = new int[m_size];
23 foreach (var cls in classes) {
24 var newid = newAlphabet.DefineClass(cls);
25 foreach (var id in cls)
26 map[id] = newid;
27 }
28
29 return map;
30 }
31
32 public int Translate(int symobl) {
33 Safe.ArgumentInRange(symobl, 0, m_size, "symbol");
34 return symobl;
35 }
36
37 public int Count {
38 get {
39 return m_size;
40 }
41 }
42
43 #endregion
44 }
45 }
46
Show file before | Show file after | Hide comments
@@ -0,0 +1,103
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4
5 namespace Implab.Automaton {
6 public class MapAlphabet<T> : IAlphabetBuilder<T> {
7 readonly Dictionary<T,int> m_map;
8 int m_nextCls;
9
10 public MapAlphabet(IEqualityComparer<T> comparer) {
11 m_map = new Dictionary<T, int>(comparer);
12 m_nextCls = 1;
13 }
14
15 #region IAlphabetBuilder implementation
16
17 public int DefineSymbol(T symbol) {
18 int cls;
19 if (m_map.TryGetValue(symbol, out cls))
20 return cls;
21
22 cls = m_nextCls++;
23
24 m_map.Add(symbol, cls);
25
26 return cls;
27 }
28
29 public int DefineClass(IEnumerable<T> symbols) {
30 Safe.ArgumentNotNull(symbols, "symbols");
31 symbols = symbols.Distinct();
32
33 foreach (var symbol in symbols) {
34 if (!m_map.Contains(symbol))
35 m_map.Add(symbol, m_nextCls);
36 else
37 throw new InvalidOperationException(String.Format("Symbol '{0}' already in use", symbol));
38 }
39 return m_nextCls++;
40 }
41
42 #endregion
43
44 #region IAlphabet implementation
45
46 public List<T>[] CreateReverseMap() {
47 var empty = new List<T>();
48 var rmap = new List<T>[m_nextCls];
49
50 for (int i = 0; i < rmap.Length; i++)
51 rmap[i] = empty;
52
53 foreach (var pair in m_map) {
54 var symbols = rmap[pair.Value];
55 if (symbols == null) {
56 symbols = new List<T>();
57 rmap[pair.Value] = symbols;
58 }
59
60 symbols.Add(pair.Key);
61 }
62
63 return rmap;
64 }
65
66 public int[] Reclassify(IAlphabetBuilder<T> newAlphabet, IEnumerable<IEnumerable<int>> classes) {
67 Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
68 Safe.ArgumentNotNull(classes, "classes");
69
70 var rmap = CreateReverseMap();
71 var map = new int[rmap.Length];
72
73 foreach (var cls in classes) {
74 var symbols = new List<T>();
75 foreach (var id in cls) {
76 if (id < 0 || id >= rmap.Length)
77 throw new ArgumentOutOfRangeException(String.Format("Class {0} is not valid for the current alphabet", id));
78 if (rmap[id] != null)
79 symbols.AddRange(rmap[id]);
80 }
81
82 var newId = newAlphabet.DefineClass(symbols);
83
84 foreach (var id in cls)
85 map[id] = newId;
86 }
87 }
88
89 public int Translate(T symobl) {
90 int cls;
91 return m_map.TryGetValue(symobl, out cls) ? cls : DFAConst.UNCLASSIFIED_INPUT;
92 }
93
94 public int Count {
95 get {
96 return m_nextCls;
97 }
98 }
99
100 #endregion
101 }
102 }
103
Show file before | Show file after | Hide comments
@@ -0,0 +1,179
1 using Implab;
2 using System;
3 using System.Collections.Generic;
4 using System.Diagnostics;
5 using System.Linq;
6
7 namespace Implab.Automaton.RegularExpressions {
8 /// <summary>
9 /// Используется для построения ДКА по регулярному выражению, сначала обходит
10 /// регулярное выражение и вычисляет followpos, затем используется метод
11 /// <see cref="BuildDFA(IDFADefinition)"/> для построения автомата.
12 /// </summary>
13 public class RegularDFABuilder<TTag> : IVisitor<TTag> {
14 int m_idx = 0;
15 Token<TTag> m_root;
16 HashSet<int> m_firstpos;
17 HashSet<int> m_lastpos;
18
19 readonly Dictionary<int, HashSet<int>> m_followpos = new Dictionary<int, HashSet<int>>();
20 readonly Dictionary<int, int> m_indexes = new Dictionary<int, int>();
21 readonly Dictionary<int, TTag> m_ends = new Dictionary<int, TTag>();
22
23 public Dictionary<int, HashSet<int>> FollowposMap {
24 get { return m_followpos; }
25 }
26
27 public HashSet<int> Followpos(int pos) {
28 HashSet<int> set;
29 if (m_followpos.TryGetValue(pos, out set))
30 return set;
31 return m_followpos[pos] = new HashSet<int>();
32 }
33
34 bool Nullable(object n) {
35 if (n is EmptyToken<TTag> || n is StarToken<TTag>)
36 return true;
37 if (n is AltToken<TTag>)
38 return Nullable(((AltToken<TTag>)n).Left) || Nullable(((AltToken<TTag>)n).Right);
39 if (n is CatToken<TTag>)
40 return Nullable(((CatToken<TTag>)n).Left) && Nullable(((CatToken<TTag>)n).Right);
41 return false;
42 }
43
44
45 public void Visit(AltToken<TTag> token) {
46 if (m_root == null)
47 m_root = token;
48 var firtspos = new HashSet<int>();
49 var lastpos = new HashSet<int>();
50
51 token.Left.Accept(this);
52 firtspos.UnionWith(m_firstpos);
53 lastpos.UnionWith(m_lastpos);
54
55 token.Right.Accept(this);
56 firtspos.UnionWith(m_firstpos);
57 lastpos.UnionWith(m_lastpos);
58
59 m_firstpos = firtspos;
60 m_lastpos = lastpos;
61 }
62
63 public void Visit(StarToken<TTag> token) {
64 if (m_root == null)
65 m_root = token;
66 token.Token.Accept(this);
67
68 foreach (var i in m_lastpos)
69 Followpos(i).UnionWith(m_firstpos);
70 }
71
72 public void Visit(CatToken<TTag> token) {
73 if (m_root == null)
74 m_root = token;
75
76 var firtspos = new HashSet<int>();
77 var lastpos = new HashSet<int>();
78 token.Left.Accept(this);
79 firtspos.UnionWith(m_firstpos);
80 var leftLastpos = m_lastpos;
81
82 token.Right.Accept(this);
83 lastpos.UnionWith(m_lastpos);
84 var rightFirstpos = m_firstpos;
85
86 if (Nullable(token.Left))
87 firtspos.UnionWith(rightFirstpos);
88
89 if (Nullable(token.Right))
90 lastpos.UnionWith(leftLastpos);
91
92 m_firstpos = firtspos;
93 m_lastpos = lastpos;
94
95 foreach (var i in leftLastpos)
96 Followpos(i).UnionWith(rightFirstpos);
97
98 }
99
100 public void Visit(EmptyToken<TTag> token) {
101 if (m_root == null)
102 m_root = token;
103 }
104
105 public void Visit(SymbolToken<TTag> token) {
106 if (m_root == null)
107 m_root = token;
108 m_idx++;
109 m_indexes[m_idx] = token.Value;
110 m_firstpos = new HashSet<int>(new[] { m_idx });
111 m_lastpos = new HashSet<int>(new[] { m_idx });
112 }
113
114 public void Visit(EndToken<TTag> token) {
115 if (m_root == null)
116 m_root = token;
117 m_idx++;
118 m_indexes[m_idx] = DFAConst.UNCLASSIFIED_INPUT;
119 m_firstpos = new HashSet<int>(new[] { m_idx });
120 m_lastpos = new HashSet<int>(new[] { m_idx });
121 Followpos(m_idx);
122 m_ends.Add(m_idx, token.Tag);
123 }
124
125 public void BuildDFA(IDFADefinitionBuilder<TTag> dfa) {
126 Safe.ArgumentNotNull(dfa,"dfa");
127
128 var states = new MapAlphabet<HashSet<int>>(new CustomEqualityComparer<HashSet<int>>(
129 (x, y) => x.SetEquals(y),
130 x => x.Sum(n => n.GetHashCode())
131 ));
132
133 var initialState = states.DefineSymbol(m_firstpos);
134
135 var tags = GetStateTags(m_firstpos);
136 if (tags != null && tags.Length > 0)
137 dfa.MarkFinalState(initialState, tags);
138
139 var inputMax = m_indexes.Values.Max();
140 var queue = new Queue<HashSet<int>>();
141
142 queue.Enqueue(m_firstpos);
143
144 while (queue.Count > 0) {
145 var state = queue.Dequeue();
146 var s1 = states.Translate(state);
147 Debug.Assert(s1 != DFAConst.UNCLASSIFIED_INPUT);
148
149 for (int a = 0; a <= inputMax; a++) {
150 var next = new HashSet<int>();
151 foreach (var p in state) {
152 if (m_indexes[p] == a) {
153 next.UnionWith(Followpos(p));
154 }
155 }
156 if (next.Count > 0) {
157 int s2 = states.Translate(next);
158 if (s2 == DFAConst.UNCLASSIFIED_INPUT) {
159 s2 = states.DefineSymbol(next);
160
161 tags = GetStateTags(next);
162 if (tags != null && tags.Length > 0)
163 dfa.MarkFinalState(s2, tags);
164
165 queue.Enqueue(next);
166 }
167 dfa.DefineTransition(s1, s2, a);
168 }
169 }
170 }
171 }
172
173 TTag[] GetStateTags(IEnumerable<int> state) {
174 Debug.Assert(state != null);
175 return state.Where(m_ends.ContainsKey).Select(pos => m_ends[pos]).ToArray();
176 }
177
178 }
179 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,17
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Threading.Tasks;
6
7 namespace Implab.JSON {
8 /// <summary>
9 /// internal
10 /// </summary>
11 public enum JSONElementContext {
12 None,
13 Object,
14 Array,
15 Closed
16 }
17 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,34
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Threading.Tasks;
6
7 namespace Implab.JSON {
8 /// <summary>
9 /// Тип элемента на котором находится парсер
10 /// </summary>
11 public enum JSONElementType {
12 None,
13 /// <summary>
14 /// Начало объекта
15 /// </summary>
16 BeginObject,
17 /// <summary>
18 /// Конец объекта
19 /// </summary>
20 EndObject,
21 /// <summary>
22 /// Начало массива
23 /// </summary>
24 BeginArray,
25 /// <summary>
26 /// Конец массива
27 /// </summary>
28 EndArray,
29 /// <summary>
30 /// Простое значение
31 /// </summary>
32 Value
33 }
34 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,99
1 using System.Linq;
2 using Implab.Automaton.RegularExpressions;
3
4 namespace Implab.Formats.JSON {
5 class JSONGrammar : Grammar<JSONGrammar> {
6 public enum TokenType {
7 None,
8 BeginObject,
9 EndObject,
10 BeginArray,
11 EndArray,
12 String,
13 Number,
14 Literal,
15 NameSeparator,
16 ValueSeparator,
17
18 StringBound,
19 EscapedChar,
20 UnescapedChar,
21 EscapedUnicode,
22
23 Minus,
24 Plus,
25 Sign,
26 Integer,
27 Dot,
28 Exp
29 }
30
31 readonly CDFADefinition m_jsonDFA;
32 readonly CDFADefinition m_stringDFA;
33
34 public JSONGrammar() {
35 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
36 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
37 var digit9 = SymbolRangeToken('1', '9');
38 var zero = SymbolToken('0');
39 var digit = zero.Or(digit9);
40 var dot = SymbolToken('.');
41 var minus = SymbolToken('-');
42 var sign = SymbolSetToken('-', '+');
43 var expSign = SymbolSetToken('e', 'E');
44 var letters = SymbolRangeToken('a', 'z');
45 var integer = zero.Or(digit9.Cat(digit.EClosure()));
46 var frac = dot.Cat(digit.Closure());
47 var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
48 var quote = SymbolToken('"');
49 var backSlash = SymbolToken('\\');
50 var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
51 var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
52 var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
53 var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
54 var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
55 var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
56 var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
57 var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
58 var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
59
60 var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
61 var literal = letters.Closure();
62 var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
63
64 var jsonExpression =
65 number.Tag(TokenType.Number)
66 .Or(literal.Tag(TokenType.Literal))
67 .Or(quote.Tag(TokenType.StringBound))
68 .Or(beginObject.Tag(TokenType.BeginObject))
69 .Or(endObject.Tag(TokenType.EndObject))
70 .Or(beginArray.Tag(TokenType.BeginArray))
71 .Or(endArray.Tag(TokenType.EndArray))
72 .Or(nameSep.Tag(TokenType.NameSeparator))
73 .Or(valueSep.Tag(TokenType.ValueSeparator));
74
75
76 var jsonStringExpression =
77 quote.Tag(TokenType.StringBound)
78 .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
79 .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
80 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
81
82
83 m_jsonDFA = BuildDFA(jsonExpression);
84 m_stringDFA = BuildDFA(jsonStringExpression);
85 }
86
87 public CDFADefinition JsonDFA {
88 get {
89 return m_jsonDFA;
90 }
91 }
92
93 public CDFADefinition JsonStringDFA {
94 get {
95 return m_stringDFA;
96 }
97 }
98 }
99 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,277
1 using Implab.Parsing;
2 using System;
3 using System.Diagnostics;
4 using System.IO;
5
6 namespace Implab.JSON {
7 /// <summary>
8 /// internal
9 /// </summary>
10 public struct JSONParserContext {
11 public string memberName;
12 public JSONElementContext elementContext;
13 }
14
15 /// <summary>
16 /// Pull парсер JSON данных.
17 /// </summary>
18 /// <remarks>
19 /// Следует отметить отдельную интерпретацию свойства <see cref="Level"/>,
20 /// оно означает текущий уровень вложенности объектов, однако закрывающий
21 /// элемент объекта и массива имеет уровень меньше, чем сам объект.
22 /// <code>
23 /// { // Level = 1
24 /// "name" : "Peter", // Level = 1
25 /// "address" : { // Level = 2
26 /// city : "Stern" // Level = 2
27 /// } // Level = 1
28 /// } // Level = 0
29 /// </code>
30 /// </remarks>
31 public class JSONParser : DFAutomaton<JSONParserContext>, IDisposable {
32
33 enum MemberContext {
34 MemberName,
35 MemberValue
36 }
37
38 static readonly EnumAlphabet<JsonTokenType> _alphabet = EnumAlphabet<JsonTokenType>.FullAlphabet;
39 static readonly DFAStateDescriptior[] _jsonDFA;
40 static readonly DFAStateDescriptior[] _objectDFA;
41 static readonly DFAStateDescriptior[] _arrayDFA;
42
43 static JSONParser() {
44
45
46 var valueExpression = Token.New(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
47 var memberExpression = Token.New(JsonTokenType.String).Cat(Token.New(JsonTokenType.NameSeparator)).Cat(valueExpression);
48
49 var objectExpression = memberExpression
50 .Cat(
51 Token.New(JsonTokenType.ValueSeparator)
52 .Cat(memberExpression)
53 .EClosure()
54 )
55 .Optional()
56 .Cat(Token.New(JsonTokenType.EndObject))
57 .Tag(0);
58 var arrayExpression = valueExpression
59 .Cat(
60 Token.New(JsonTokenType.ValueSeparator)
61 .Cat(valueExpression)
62 .EClosure()
63 )
64 .Optional()
65 .Cat(Token.New(JsonTokenType.EndArray))
66 .Tag(0);
67
68 var jsonExpression = valueExpression.Tag(0);
69
70 _jsonDFA = BuildDFA(jsonExpression).States;
71 _objectDFA = BuildDFA(objectExpression).States;
72 _arrayDFA = BuildDFA(arrayExpression).States;
73 }
74
75 static EDFADefinition<JsonTokenType> BuildDFA(Token expr) {
76 var builder = new DFABuilder();
77 var dfa = new EDFADefinition<JsonTokenType>(_alphabet);
78 expr.Accept(builder);
79
80 builder.BuildDFA(dfa);
81 return dfa;
82 }
83
84 JSONScanner m_scanner;
85 MemberContext m_memberContext;
86
87 JSONElementType m_elementType;
88 object m_elementValue;
89
90 /// <summary>
91 /// Создает новый парсер на основе строки, содержащей JSON
92 /// </summary>
93 /// <param name="text"></param>
94 public JSONParser(string text)
95 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
96 Safe.ArgumentNotEmpty(text, "text");
97 m_scanner = new JSONScanner();
98 m_scanner.Feed(text.ToCharArray());
99 }
100
101 /// <summary>
102 /// Создает новый экземпляр парсера, на основе текстового потока.
103 /// </summary>
104 /// <param name="reader">Текстовый поток.</param>
105 /// <param name="dispose">Признак того, что парсер должен конролировать время жизни входного потока.</param>
106 public JSONParser(TextReader reader, bool dispose)
107 : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
108 Safe.ArgumentNotNull(reader, "reader");
109 m_scanner = new JSONScanner();
110 m_scanner.Feed(reader, dispose);
111 }
112
113 /// <summary>
114 /// Тип текущего элемента на котором стоит парсер.
115 /// </summary>
116 public JSONElementType ElementType {
117 get { return m_elementType; }
118 }
119
120 /// <summary>
121 /// Имя элемента - имя свойства родительского контейнера. Для элементов массивов и корневого всегда
122 /// пустая строка.
123 /// </summary>
124 public string ElementName {
125 get { return m_context.info.memberName; }
126 }
127
128 /// <summary>
129 /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c>
130 /// </summary>
131 public object ElementValue {
132 get { return m_elementValue; }
133 }
134
135 /// <summary>
136 /// Читает слеюудущий объект из потока
137 /// </summary>
138 /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
139 public bool Read() {
140 if (m_context.current == UNREACHEBLE_STATE)
141 throw new InvalidOperationException("The parser is in invalid state");
142 object tokenValue;
143 JsonTokenType tokenType;
144 m_context.info.memberName = String.Empty;
145 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
146 Move((int)tokenType);
147 if (m_context.current == UNREACHEBLE_STATE)
148 UnexpectedToken(tokenValue, tokenType);
149 switch (tokenType) {
150 case JsonTokenType.BeginObject:
151 Switch(
152 _objectDFA,
153 INITIAL_STATE,
154 new JSONParserContext {
155 memberName = m_context.info.memberName,
156 elementContext = JSONElementContext.Object
157 }
158 );
159 m_elementValue = null;
160 m_memberContext = MemberContext.MemberName;
161 m_elementType = JSONElementType.BeginObject;
162 return true;
163 case JsonTokenType.EndObject:
164 Restore();
165 m_elementValue = null;
166 m_elementType = JSONElementType.EndObject;
167 return true;
168 case JsonTokenType.BeginArray:
169 Switch(
170 _arrayDFA,
171 INITIAL_STATE,
172 new JSONParserContext {
173 memberName = m_context.info.memberName,
174 elementContext = JSONElementContext.Array
175 }
176 );
177 m_elementValue = null;
178 m_memberContext = MemberContext.MemberValue;
179 m_elementType = JSONElementType.BeginArray;
180 return true;
181 case JsonTokenType.EndArray:
182 Restore();
183 m_elementValue = null;
184 m_elementType = JSONElementType.EndArray;
185 return true;
186 case JsonTokenType.String:
187 if (m_memberContext == MemberContext.MemberName) {
188 m_context.info.memberName = (string)tokenValue;
189 break;
190 }
191 m_elementType = JSONElementType.Value;
192 m_elementValue = tokenValue;
193 return true;
194 case JsonTokenType.Number:
195 m_elementType = JSONElementType.Value;
196 m_elementValue = tokenValue;
197 return true;
198 case JsonTokenType.Literal:
199 m_elementType = JSONElementType.Value;
200 m_elementValue = ParseLiteral((string)tokenValue);
201 return true;
202 case JsonTokenType.NameSeparator:
203 m_memberContext = MemberContext.MemberValue;
204 break;
205 case JsonTokenType.ValueSeparator:
206 m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
207 break;
208 default:
209 UnexpectedToken(tokenValue, tokenType);
210 break;
211 }
212 }
213 if (m_context.info.elementContext != JSONElementContext.None)
214 throw new ParserException("Unexpedted end of data");
215 return false;
216 }
217
218 object ParseLiteral(string literal) {
219 switch (literal) {
220 case "null":
221 return null;
222 case "false":
223 return false;
224 case "true":
225 return true;
226 default:
227 UnexpectedToken(literal, JsonTokenType.Literal);
228 return null; // avoid compliler error
229 }
230 }
231
232 void UnexpectedToken(object value, JsonTokenType tokenType) {
233 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
234 }
235
236
237 /// <summary>
238 /// Признак конца потока
239 /// </summary>
240 public bool EOF {
241 get {
242 return m_scanner.EOF;
243 }
244 }
245
246 protected virtual void Dispose(bool disposing) {
247 if (disposing) {
248 m_scanner.Dispose();
249 }
250 }
251
252 /// <summary>
253 /// Освобождает парсер и связанный с ним сканнер.
254 /// </summary>
255 public void Dispose() {
256 Dispose(true);
257 GC.SuppressFinalize(this);
258 }
259
260 ~JSONParser() {
261 Dispose(false);
262 }
263
264 /// <summary>
265 /// Переходит в конец текущего объекта.
266 /// </summary>
267 public void SeekElementEnd() {
268 var level = Level - 1;
269
270 Debug.Assert(level >= 0);
271
272 while (Level != level)
273 Read();
274 }
275 }
276
277 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,100
1 using Implab.Parsing;
2 using System;
3 using System.Collections.Generic;
4 using System.Globalization;
5 using System.Linq;
6 using System.Text;
7 using System.Threading.Tasks;
8
9 namespace Implab.JSON {
10 /// <summary>
11 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
12 /// </summary>
13 public class JSONScanner : Scanner {
14 char[] m_stringBuffer;
15 DFAStateDescriptior[] m_stringDFA;
16 int[] m_stringAlphabet;
17
18 /// <summary>
19 /// Создает новый экземпляр сканнера
20 /// </summary>
21 public JSONScanner()
22 : base(JSONGrammar.Instance.JsonDFA.States, JSONGrammar.Instance.JsonDFA.Alphabet.GetTranslationMap()) {
23 m_stringBuffer = new char[1024];
24 var dfa = JSONGrammar.Instance.JsonStringDFA;
25 m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
26 m_stringDFA = dfa.States;
27 }
28
29 /// <summary>
30 /// Читает следующий лексический элемент из входных данных.
31 /// </summary>
32 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
33 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
34 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
35 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
36 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
37 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
38 if (ReadTokenInternal()) {
39 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
40 case JSONGrammar.TokenType.StringBound:
41 tokenValue = ReadString();
42 tokenType = JsonTokenType.String;
43 break;
44 case JSONGrammar.TokenType.Number:
45 tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
46 tokenType = JsonTokenType.Number;
47 break;
48 default:
49 tokenType = (JsonTokenType)m_currentState.tag[0];
50 tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
51 break;
52 }
53 return true;
54 }
55 tokenValue = null;
56 tokenType = JsonTokenType.None;
57 return false;
58 }
59
60 string ReadString() {
61 int pos = 0;
62 Switch(m_stringDFA, m_stringAlphabet);
63 while (ReadTokenInternal()) {
64 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
65 case JSONGrammar.TokenType.StringBound:
66 Restore();
67 return new String(m_stringBuffer, 0, pos);
68 case JSONGrammar.TokenType.UnescapedChar:
69 EnsureStringBufferSize(pos + m_tokenLen);
70 Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
71 pos += m_tokenLen;
72 break;
73 case JSONGrammar.TokenType.EscapedUnicode:
74 EnsureStringBufferSize(pos + 1);
75 m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
76 pos++;
77 break;
78 case JSONGrammar.TokenType.EscapedChar:
79 EnsureStringBufferSize(pos + 1);
80 m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
81 pos++;
82 break;
83 default:
84 break;
85 }
86
87 }
88
89 throw new ParserException("Unexpected end of data");
90 }
91
92 void EnsureStringBufferSize(int size) {
93 if (size > m_stringBuffer.Length) {
94 var newBuffer = new char[size];
95 m_stringBuffer.CopyTo(newBuffer, 0);
96 m_stringBuffer = newBuffer;
97 }
98 }
99 }
100 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,319
1 using System;
2 using System.Collections.Generic;
3 using System.IO;
4 using System.Globalization;
5 using System.Diagnostics;
6
7 namespace Implab.JSON {
8 public class JSONWriter {
9 struct Context {
10 public bool needComma;
11 public JSONElementContext element;
12 }
13 Stack<Context> m_contextStack = new Stack<Context>();
14 Context m_context;
15
16 const int BUFFER_SIZE = 64;
17
18 TextWriter m_writer;
19 readonly bool m_indent = true;
20 readonly int m_indentSize = 4;
21 readonly char[] m_buffer = new char[BUFFER_SIZE];
22 int m_bufferPos;
23
24 static readonly char [] _hex = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
25 static readonly char [] _escapeBKS,
26 _escapeFWD,
27 _escapeCR,
28 _escapeNL,
29 _escapeTAB,
30 _escapeBSLASH,
31 _escapeQ;
32
33 static JSONWriter() {
34 _escapeBKS = "\\b".ToCharArray();
35 _escapeFWD = "\\f".ToCharArray();
36 _escapeCR = "\\r".ToCharArray();
37 _escapeNL = "\\n".ToCharArray();
38 _escapeTAB = "\\t".ToCharArray();
39 _escapeBSLASH = "\\\\".ToCharArray();
40 _escapeQ = "\\\"".ToCharArray();
41 }
42
43 public JSONWriter(TextWriter writer) {
44 Safe.ArgumentNotNull(writer, "writer");
45 m_writer = writer;
46 }
47
48 public JSONWriter(TextWriter writer, bool indent) {
49 Safe.ArgumentNotNull(writer, "writer");
50
51 m_writer = writer;
52 m_indent = indent;
53 }
54
55 void WriteIndent() {
56 if (m_indent) {
57 var indent = new char[m_contextStack.Count * m_indentSize + 1];
58 indent[0] = '\n';
59 for (int i = 1; i < indent.Length; i++)
60 indent[i] = ' ';
61 m_writer.Write(new String(indent));
62 } else {
63 m_writer.Write(' ');
64 }
65 }
66
67 void WriteMemberName(string name) {
68 Safe.ArgumentNotEmpty(name, "name");
69 if (m_context.element != JSONElementContext.Object)
70 OperationNotApplicable("WriteMember");
71 if (m_context.needComma)
72 m_writer.Write(",");
73
74 WriteIndent();
75 m_context.needComma = true;
76 Write(name);
77 m_writer.Write(" : ");
78 }
79
80 public void WriteValue(string name, string value) {
81 WriteMemberName(name);
82 Write(value);
83 }
84
85 public void WriteValue(string name, bool value) {
86 WriteMemberName(name);
87 Write(value);
88 }
89
90 public void WriteValue(string name, double value) {
91 WriteMemberName(name);
92 Write(value);
93 }
94
95 public void WriteValue(string value) {
96 if (m_context.element == JSONElementContext.Array) {
97
98 if (m_context.needComma)
99 m_writer.Write(",");
100 WriteIndent();
101 m_context.needComma = true;
102
103 Write(value);
104 } else if (m_context.element == JSONElementContext.None) {
105 Write(value);
106 m_context.element = JSONElementContext.Closed;
107 } else {
108 OperationNotApplicable("WriteValue");
109 }
110 }
111
112 public void WriteValue(bool value) {
113 if (m_context.element == JSONElementContext.Array) {
114
115 if (m_context.needComma)
116 m_writer.Write(",");
117 WriteIndent();
118 m_context.needComma = true;
119
120 Write(value);
121 } else if (m_context.element == JSONElementContext.None) {
122 Write(value);
123 m_context.element = JSONElementContext.Closed;
124 } else {
125 OperationNotApplicable("WriteValue");
126 }
127 }
128
129 public void WriteValue(double value) {
130 if (m_context.element == JSONElementContext.Array) {
131
132 if (m_context.needComma)
133 m_writer.Write(",");
134 WriteIndent();
135 m_context.needComma = true;
136
137 Write(value);
138 } else if (m_context.element == JSONElementContext.None) {
139 Write(value);
140 m_context.element = JSONElementContext.Closed;
141 } else {
142 OperationNotApplicable("WriteValue");
143 }
144 }
145
146 public void BeginObject() {
147 if (m_context.element != JSONElementContext.None && m_context.element != JSONElementContext.Array)
148 OperationNotApplicable("BeginObject");
149 if (m_context.needComma)
150 m_writer.Write(",");
151
152 WriteIndent();
153
154 m_context.needComma = true;
155
156 m_contextStack.Push(m_context);
157
158 m_context = new Context { element = JSONElementContext.Object, needComma = false };
159 m_writer.Write("{");
160 }
161
162 public void BeginObject(string name) {
163 WriteMemberName(name);
164
165 m_contextStack.Push(m_context);
166
167 m_context = new Context { element = JSONElementContext.Object, needComma = false };
168 m_writer.Write("{");
169 }
170
171 public void EndObject() {
172 if (m_context.element != JSONElementContext.Object)
173 OperationNotApplicable("EndObject");
174
175 m_context = m_contextStack.Pop();
176 if (m_contextStack.Count == 0)
177 m_context.element = JSONElementContext.Closed;
178 WriteIndent();
179 m_writer.Write("}");
180 }
181
182 public void BeginArray() {
183 if (m_context.element != JSONElementContext.None && m_context.element != JSONElementContext.Array)
184 throw new InvalidOperationException();
185 if (m_context.needComma) {
186 m_writer.Write(",");
187
188 }
189 m_context.needComma = true;
190
191 WriteIndent();
192 m_contextStack.Push(m_context);
193 m_context = new Context { element = JSONElementContext.Array, needComma = false };
194 m_writer.Write("[");
195 }
196
197 public void BeginArray(string name) {
198 WriteMemberName(name);
199
200 m_contextStack.Push(m_context);
201
202 m_context = new Context { element = JSONElementContext.Array, needComma = false };
203 m_writer.Write("[");
204 }
205
206 public void EndArray() {
207 if (m_context.element != JSONElementContext.Array)
208 OperationNotApplicable("EndArray");
209
210 m_context = m_contextStack.Pop();
211 if (m_contextStack.Count == 0)
212 m_context.element = JSONElementContext.Closed;
213 WriteIndent();
214 m_writer.Write("]");
215 }
216
217 void Write(bool value) {
218 m_writer.Write(value ? "true" : "false");
219 }
220
221 void FlushBuffer() {
222 if (m_bufferPos > 0) {
223 m_writer.Write(m_buffer, 0, m_bufferPos);
224 m_bufferPos = 0;
225 }
226 }
227
228 void Write(string value) {
229 if (value == null) {
230 m_writer.Write("null");
231 return;
232 }
233
234 Debug.Assert(m_bufferPos == 0);
235
236 var chars = value.ToCharArray();
237 m_buffer[m_bufferPos++] = '"';
238
239 // Analysis disable once ForCanBeConvertedToForeach
240 for (int i = 0; i < chars.Length; i++) {
241 var ch = chars[i];
242
243 char[] escapeSeq;
244
245 switch (ch) {
246 case '\b':
247 escapeSeq = _escapeBKS;
248 break;
249 case '\f':
250 escapeSeq = _escapeFWD;
251 break;
252 case '\r':
253 escapeSeq = _escapeCR;
254 break;
255 case '\n':
256 escapeSeq = _escapeNL;
257 break;
258 case '\t':
259 escapeSeq = _escapeTAB;
260 break;
261 case '\\':
262 escapeSeq = _escapeBSLASH;
263 break;
264 case '"':
265 escapeSeq = _escapeQ;
266 break;
267 default:
268 if (ch < 0x20) {
269 if (m_bufferPos + 6 > BUFFER_SIZE)
270 FlushBuffer();
271
272 m_buffer[m_bufferPos++] = '\\';
273 m_buffer[m_bufferPos++] = 'u';
274 m_buffer[m_bufferPos++] = '0';
275 m_buffer[m_bufferPos++] = '0';
276 m_buffer[m_bufferPos++] = _hex[ch >> 4 & 0xf];
277 m_buffer[m_bufferPos++] = _hex[ch & 0xf];
278
279 } else {
280 if (m_bufferPos >= BUFFER_SIZE)
281 FlushBuffer();
282 m_buffer[m_bufferPos++] = ch;
283 }
284 continue;
285 }
286
287 if (m_bufferPos + escapeSeq.Length > BUFFER_SIZE)
288 FlushBuffer();
289
290 Array.Copy(escapeSeq, 0, m_buffer, m_bufferPos, escapeSeq.Length);
291 m_bufferPos += escapeSeq.Length;
292
293 }
294
295 if (m_bufferPos >= BUFFER_SIZE)
296 FlushBuffer();
297
298 m_buffer[m_bufferPos++] = '"';
299
300 FlushBuffer();
301 }
302
303 void Write(double value) {
304 if (double.IsNaN(value))
305 Write("NaN");
306 else if (double.IsNegativeInfinity(value))
307 Write("-Infinity");
308 else if (double.IsPositiveInfinity(value))
309 Write("Infinity");
310 else
311 m_writer.Write(value.ToString(CultureInfo.InvariantCulture));
312 }
313
314 void OperationNotApplicable(string opName) {
315 throw new InvalidOperationException(String.Format("The operation '{0}' isn't applicable in the context of '{1}'", opName, m_context.element ));
316 }
317
318 }
319 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,343
1 using Implab;
2 using Implab.Parsing;
3 using System;
4 using System.Collections.Generic;
5 using System.Globalization;
6 using System.IO;
7 using System.Linq;
8 using System.Text;
9 using System.Threading.Tasks;
10 using System.Xml;
11
12 namespace Implab.JSON {
13 public class JSONXmlReader : XmlReader {
14
15 enum ValueContext {
16 Undefined,
17 ElementStart,
18 ElementValue,
19 ElementEnd,
20 ElementEmpty
21 }
22
23 struct LocalNameContext {
24 public string localName;
25 public bool isArray;
26 }
27
28 JSONParser m_parser;
29 ValueContext m_valueContext;
30 ReadState m_state = ReadState.Initial;
31 Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
32 LocalNameContext m_localName;
33 int m_depthCorrection = 0;
34
35 readonly string m_rootName;
36 readonly string m_prefix;
37 readonly string m_namespaceUri;
38 readonly bool m_flattenArrays;
39 readonly string m_arrayItemName;
40 readonly XmlNameTable m_nameTable;
41
42 JSONXmlReader(JSONParser parser, JSONXmlReaderOptions options) {
43 m_parser = parser;
44
45 if (options != null) {
46 m_prefix = options.NodesPrefix ?? String.Empty;
47 m_namespaceUri = options.NamespaceURI ?? String.Empty;
48 m_rootName = options.RootName ?? "json";
49 m_flattenArrays = options.FlattenArrays;
50 m_arrayItemName = options.ArrayItemName ?? "item";
51 m_nameTable = options.NameTable ?? new NameTable();
52 } else {
53 m_prefix = String.Empty;
54 m_namespaceUri = String.Empty;
55 m_rootName = "json";
56 m_flattenArrays = false;
57 m_arrayItemName = "item";
58 m_nameTable = new NameTable();
59 }
60 }
61
62 /// <summary>
63 /// Always 0, JSON doesn't support attributes
64 /// </summary>
65 public override int AttributeCount {
66 get { return 0; }
67 }
68
69 public override string BaseURI {
70 get { return String.Empty; }
71 }
72
73 public override int Depth {
74 get {
75 return m_localNameStack.Count + m_depthCorrection;
76 }
77 }
78
79 public override bool EOF {
80 get { return m_parser.EOF; }
81 }
82
83 /// <summary>
84 /// Always throws an exception
85 /// </summary>
86 /// <param name="i"></param>
87 /// <returns></returns>
88 public override string GetAttribute(int i) {
89 throw new ArgumentOutOfRangeException();
90 }
91
92 /// <summary>
93 /// Always returns empty string
94 /// </summary>
95 /// <param name="name"></param>
96 /// <param name="namespaceURI"></param>
97 /// <returns></returns>
98 public override string GetAttribute(string name, string namespaceURI) {
99 return String.Empty;
100 }
101
102 /// <summary>
103 /// Always returns empty string
104 /// </summary>
105 /// <param name="name"></param>
106 /// <returns></returns>
107 public override string GetAttribute(string name) {
108 return String.Empty;
109 }
110
111 public override bool IsEmptyElement {
112 get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
113 }
114
115 public override string LocalName {
116 get { return m_localName.localName; }
117 }
118
119 public override string LookupNamespace(string prefix) {
120 if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
121 return m_namespaceUri;
122 else
123 return String.Empty;
124 }
125
126 public override bool MoveToAttribute(string name, string ns) {
127 return false;
128 }
129
130 public override bool MoveToAttribute(string name) {
131 return false;
132 }
133
134 public override bool MoveToElement() {
135 return false;
136 }
137
138 public override bool MoveToFirstAttribute() {
139 return false;
140 }
141
142 public override bool MoveToNextAttribute() {
143 return false;
144 }
145
146 public override XmlNameTable NameTable {
147 get { return m_nameTable; }
148 }
149
150 public override string NamespaceURI {
151 get { return m_namespaceUri; }
152 }
153
154 public override XmlNodeType NodeType {
155 get {
156 switch (m_parser.ElementType) {
157 case JSONElementType.BeginObject:
158 case JSONElementType.BeginArray:
159 return XmlNodeType.Element;
160 case JSONElementType.EndObject:
161 case JSONElementType.EndArray:
162 return XmlNodeType.EndElement;
163 case JSONElementType.Value:
164 switch (m_valueContext) {
165 case ValueContext.ElementStart:
166 case ValueContext.ElementEmpty:
167 return XmlNodeType.Element;
168 case ValueContext.ElementValue:
169 return XmlNodeType.Text;
170 case ValueContext.ElementEnd:
171 return XmlNodeType.EndElement;
172 default:
173 throw new InvalidOperationException();
174 }
175 default:
176 throw new InvalidOperationException();
177 }
178 }
179 }
180
181 public override string Prefix {
182 get { return m_prefix; }
183 }
184
185 public override bool Read() {
186 if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
187 return false;
188
189 if (m_state == ReadState.Initial)
190 m_state = System.Xml.ReadState.Interactive;
191
192 try {
193 switch (m_parser.ElementType) {
194 case JSONElementType.Value:
195 switch (m_valueContext) {
196 case ValueContext.ElementStart:
197 SetLocalName(String.Empty);
198 m_valueContext = ValueContext.ElementValue;
199 return true;
200 case ValueContext.ElementValue:
201 RestoreLocalName();
202 m_valueContext = ValueContext.ElementEnd;
203 return true;
204 case ValueContext.ElementEmpty:
205 case ValueContext.ElementEnd:
206 RestoreLocalName();
207 break;
208 }
209 break;
210 case JSONElementType.EndArray:
211 case JSONElementType.EndObject:
212 RestoreLocalName();
213 break;
214 }
215 string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : m_arrayItemName;
216 while (m_parser.Read()) {
217 if (!String.IsNullOrEmpty(m_parser.ElementName))
218 itemName = m_parser.ElementName;
219
220 switch (m_parser.ElementType) {
221 case JSONElementType.BeginArray:
222 if (m_flattenArrays && !m_localName.isArray) {
223 m_depthCorrection--;
224 SetLocalName(itemName, true);
225 continue;
226 } else {
227 SetLocalName(itemName, true);
228 }
229 break;
230 case JSONElementType.BeginObject:
231 SetLocalName(itemName);
232 break;
233 case JSONElementType.EndArray:
234 if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
235 RestoreLocalName();
236 m_depthCorrection++;
237 continue;
238 }
239 break;
240 case JSONElementType.EndObject:
241 break;
242 case JSONElementType.Value:
243 SetLocalName(itemName);
244 m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
245 break;
246 default:
247 break;
248 }
249 return true;
250 }
251
252 m_state = System.Xml.ReadState.EndOfFile;
253 return false;
254 } catch {
255 m_state = System.Xml.ReadState.Error;
256 throw;
257 }
258 }
259
260 public override bool ReadAttributeValue() {
261 return false;
262 }
263
264 public override ReadState ReadState {
265 get { return m_state; }
266 }
267
268 public override void ResolveEntity() {
269 // do nothing
270 }
271
272 public override string Value {
273 get {
274 if (m_parser.ElementValue == null)
275 return String.Empty;
276 if (Convert.GetTypeCode(m_parser.ElementValue) == TypeCode.Double)
277 return ((double)m_parser.ElementValue).ToString(CultureInfo.InvariantCulture);
278 else
279 return m_parser.ElementValue.ToString();
280 }
281 }
282
283 void SetLocalName(string name) {
284 m_localNameStack.Push(m_localName);
285 m_localName.localName = name;
286 m_localName.isArray = false;
287 }
288
289 void SetLocalName(string name, bool isArray) {
290 m_localNameStack.Push(m_localName);
291 m_localName.localName = name;
292 m_localName.isArray = isArray;
293 }
294
295 void RestoreLocalName() {
296 m_localName = m_localNameStack.Pop();
297 }
298
299 public override void Close() {
300
301 }
302
303 protected override void Dispose(bool disposing) {
304 #if MONO
305 disposing = true;
306 #endif
307 if (disposing) {
308 m_parser.Dispose();
309 }
310 base.Dispose(disposing);
311 }
312
313 public static JSONXmlReader Create(string file, JSONXmlReaderOptions options) {
314 return Create(File.OpenText(file), options);
315 }
316
317 /// <summary>
318 /// Creates the XmlReader for the specified text stream with JSON data.
319 /// </summary>
320 /// <param name="reader">Text reader.</param>
321 /// <param name="options">Options.</param>
322 /// <remarks>
323 /// The reader will be disposed when the XmlReader is disposed.
324 /// </remarks>
325 public static JSONXmlReader Create(TextReader reader, JSONXmlReaderOptions options) {
326 return new JSONXmlReader(new JSONParser(reader, true), options);
327 }
328
329 /// <summary>
330 /// Creates the XmlReader for the specified stream with JSON data.
331 /// </summary>
332 /// <param name="stream">Stream.</param>
333 /// <param name="options">Options.</param>
334 /// <remarks>
335 /// The stream will be disposed when the XmlReader is disposed.
336 /// </remarks>
337 public static JSONXmlReader Create(Stream stream, JSONXmlReaderOptions options) {
338 Safe.ArgumentNotNull(stream, "stream");
339 // HACK don't dispose StreaReader to keep stream opened
340 return Create(new StreamReader(stream), options);
341 }
342 }
343 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,65
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Xml;
6
7 namespace Implab.JSON {
8 /// <summary>
9 /// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
10 /// интерпретации <c>JSON</c> документа.
11 /// </summary>
12 public class JSONXmlReaderOptions {
13 /// <summary>
14 /// Пространство имен в котором будут располагаться читаемые элементы документа
15 /// </summary>
16 public string NamespaceURI {
17 get;
18 set;
19 }
20
21 /// <summary>
22 /// Интерпретировать массивы как множественные элементы (убирает один уровень вложенности), иначе массив
23 /// представляется в виде узла, дочерними элементами которого являются элементы массива, имена дочерних элементов
24 /// определяются свойством <see cref="ArrayItemName"/>. По умолчанию <c>false</c>.
25 /// </summary>
26 public bool FlattenArrays {
27 get;
28 set;
29 }
30
31 /// <summary>
32 /// Префикс, для узлов документа
33 /// </summary>
34 public string NodesPrefix {
35 get;
36 set;
37 }
38
39 /// <summary>
40 /// Имя корневого элемента в xml документе
41 /// </summary>
42 public string RootName {
43 get;
44 set;
45 }
46
47 /// <summary>
48 /// Имя элемента для массивов, если не включена опция <see cref="FlattenArrays"/>.
49 /// По умолчанию <c>item</c>.
50 /// </summary>
51 public string ArrayItemName {
52 get;
53 set;
54 }
55
56 /// <summary>
57 /// Таблица атомизированных строк для построения документа.
58 /// </summary>
59 public XmlNameTable NameTable {
60 get;
61 set;
62 }
63
64 }
65 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,50
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Threading.Tasks;
6
7 namespace Implab.JSON {
8 /// <summary>
9 /// Тип токенов, возвращаемых <see cref="JSONScanner"/>.
10 /// </summary>
11 public enum JsonTokenType : int {
12 None = 0,
13 /// <summary>
14 /// Начало объекта
15 /// </summary>
16 BeginObject,
17 /// <summary>
18 /// Конец объекта
19 /// </summary>
20 EndObject,
21 /// <summary>
22 /// Начало массива
23 /// </summary>
24 BeginArray,
25 /// <summary>
26 /// Конец массива
27 /// </summary>
28 EndArray,
29 /// <summary>
30 /// Строка
31 /// </summary>
32 String,
33 /// <summary>
34 /// Число
35 /// </summary>
36 Number,
37 /// <summary>
38 /// Литерал
39 /// </summary>
40 Literal,
41 /// <summary>
42 /// Разделитель имени <c>:</c>
43 /// </summary>
44 NameSeparator,
45 /// <summary>
46 /// Разделитель имени <c>,</c>
47 /// </summary>
48 ValueSeparator
49 }
50 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,96
1 using Implab;
2 using Implab.Parsing;
3 using System;
4 using System.Collections.Generic;
5 using System.Diagnostics;
6 using System.Linq;
7 using System.Text;
8 using System.Threading.Tasks;
9
10 namespace Implab.JSON {
11 /// <summary>
12 /// Класс для преобразования экранированной строки JSON
13 /// </summary>
14 public class StringTranslator : Scanner {
15 static readonly char[] _escMap;
16 static readonly int[] _hexMap;
17
18 static StringTranslator() {
19 var chars = new char[] { 'b', 'f', 't', 'r', 'n', '\\', '/' };
20 var vals = new char[] { '\b', '\f', '\t', '\r', '\n', '\\', '/' };
21
22 _escMap = new char[chars.Max() + 1];
23
24 for (int i = 0; i < chars.Length; i++)
25 _escMap[chars[i]] = vals[i];
26
27 var hexs = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' };
28 var ints = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 10, 11, 12, 13, 14, 15 };
29
30 _hexMap = new int[hexs.Max() + 1];
31
32 for (int i = 0; i < hexs.Length; i++)
33 _hexMap[hexs[i]] = ints[i];
34
35 }
36
37 public StringTranslator()
38 : base(JSONGrammar.Instance.JsonStringDFA.States, JSONGrammar.Instance.JsonStringDFA.Alphabet.GetTranslationMap()) {
39 }
40
41 public string Translate(string data) {
42 Safe.ArgumentNotNull(data, "data");
43 return Translate(data.ToCharArray());
44 }
45
46 public string Translate(char[] data) {
47 Safe.ArgumentNotNull(data, "data");
48 return Translate(data, data.Length);
49 }
50
51 public string Translate(char[] data, int length) {
52 Safe.ArgumentNotNull(data, "data");
53 Safe.ArgumentInRange(length, 0, data.Length, "length");
54
55 var translated = new char[length];
56
57 Feed(data,length);
58
59 int pos = 0;
60
61 while (ReadTokenInternal()) {
62 switch ((JSONGrammar.TokenType)TokenTags[0]) {
63 case JSONGrammar.TokenType.UnescapedChar:
64 Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
65 pos += m_tokenLen;
66 break;
67 case JSONGrammar.TokenType.EscapedChar:
68 translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
69 pos++;
70 break;
71 case JSONGrammar.TokenType.EscapedUnicode:
72 translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
73 pos++;
74 break;
75 }
76 }
77
78 return new String(translated, 0, pos);
79 }
80
81 internal static char TranslateEscapedChar(char symbol) {
82 return _escMap[symbol];
83 }
84
85 internal static char TranslateHexUnicode(char[] symbols, int offset) {
86 Debug.Assert(symbols != null);
87 Debug.Assert(symbols.Length - offset >= 4);
88
89 int value = (_hexMap[symbols[offset]] << 12)
90 | (_hexMap[symbols[offset + 1]] << 8)
91 | (_hexMap[symbols[offset + 2]] << 4)
92 | (_hexMap[symbols[offset + 3]]);
93 return (char)value;
94 }
95 }
96 }
Show file before | Show file after | Hide comments
@@ -9,86 +9,71 namespace Implab.Automaton.RegularExpres
9 9 /// <summary>
10 10 /// Базовый абстрактный класс. Грамматика, позволяет формулировать выражения над алфавитом типа <c>char</c>.
11 11 /// </summary>
12 /// <typeparam name="TGrammar"></typeparam>
13 public abstract class Grammar<TGrammar> where TGrammar: Grammar<TGrammar>, new() {
14 static TGrammar _instance;
12 public abstract class Grammar<TSymbol, TTag> {
15 13
16 public static TGrammar Instance{
17 get {
18 if (_instance == null)
19 _instance = new TGrammar();
20 return _instance;
21 }
14 public abstract IAlphabetBuilder<TSymbol> Alphabet {
15 get;
22 16 }
23 17
24 readonly CharAlphabet m_alphabet = new CharAlphabet();
25
26 public CharAlphabet Alphabet {
27 get { return m_alphabet; }
18 public SymbolToken<TTag> UnclassifiedToken() {
19 return new SymbolToken<TTag>(DFAConst.UNCLASSIFIED_INPUT);
28 20 }
29 21
30 public SymbolToken UnclassifiedToken() {
31 return new SymbolToken(CharAlphabet.UNCLASSIFIED);
32 }
33
34 public void DefineAlphabet(IEnumerable<char> alphabet) {
22 public void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
35 23 Safe.ArgumentNotNull(alphabet, "alphabet");
36 24
37 25 foreach (var ch in alphabet)
38 m_alphabet.DefineSymbol(ch);
26 Alphabet.DefineSymbol(ch);
39 27 }
40 public Token SymbolRangeToken(char start, char end) {
41 return SymbolToken(Enumerable.Range(start, end - start + 1).Select(x => (char)x));
28
29 public Token<TTag> SymbolToken(TSymbol symbol) {
30 return Token<TTag>.New(TranslateOrAdd(symbol));
42 31 }
43 32
44 public Token SymbolToken(char symbol) {
45 return Token.New(TranslateOrAdd(symbol));
33 public Token<TTag> SymbolToken(IEnumerable<TSymbol> symbols) {
34 Safe.ArgumentNotNull(symbols, "symbols");
35
36 return Token<TTag>.New(TranslateOrAdd(symbols).ToArray());
46 37 }
47 38
48 public Token SymbolToken(IEnumerable<char> symbols) {
49 Safe.ArgumentNotNull(symbols, "symbols");
50
51 return Token.New(TranslateOrAdd(symbols).ToArray());
52 }
53
54 public Token SymbolSetToken(params char[] set) {
39 public Token<TTag> SymbolSetToken(params TSymbol[] set) {
55 40 return SymbolToken(set);
56 41 }
57 42
58 int TranslateOrAdd(char ch) {
59 var t = m_alphabet.Translate(ch);
60 if (t == CharAlphabet.UNCLASSIFIED)
61 t = m_alphabet.DefineSymbol(ch);
43 int TranslateOrAdd(TSymbol ch) {
44 var t = Alphabet.Translate(ch);
45 if (t == DFAConst.UNCLASSIFIED_INPUT)
46 t = Alphabet.DefineSymbol(ch);
62 47 return t;
63 48 }
64 49
65 IEnumerable<int> TranslateOrAdd(IEnumerable<char> symbols) {
50 IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
66 51 return symbols.Distinct().Select(TranslateOrAdd);
67 52 }
68 53
69 int TranslateOrDie(char ch) {
70 var t = m_alphabet.Translate(ch);
71 if (t == CharAlphabet.UNCLASSIFIED)
54 int TranslateOrDie(TSymbol ch) {
55 var t = Alphabet.Translate(ch);
56 if (t == DFAConst.UNCLASSIFIED_INPUT)
72 57 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
73 58 return t;
74 59 }
75 60
76 IEnumerable<int> TranslateOrDie(IEnumerable<char> symbols) {
61 IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
77 62 return symbols.Distinct().Select(TranslateOrDie);
78 63 }
79 64
80 public Token SymbolTokenExcept(IEnumerable<char> symbols) {
65 public Token<TTag> SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
81 66 Safe.ArgumentNotNull(symbols, "symbols");
82 67
83 return Token.New( Enumerable.Range(0, m_alphabet.Count).Except(TranslateOrDie(symbols)).ToArray());
68 return Token<TTag>.New( Enumerable.Range(0, Alphabet.Count).Except(TranslateOrDie(symbols)).ToArray() );
84 69 }
85 70
86 protected CDFADefinition BuildDFA(Token lang) {
71 protected CDFADefinition BuildDFA(Token<TTag> lang) {
87 72 Safe.ArgumentNotNull(lang, "lang");
88 73
89 var dfa = new CDFADefinition(m_alphabet);
74 var dfa = new CDFADefinition(Alphabet);
90 75
91 var builder = new DFABuilder();
76 var builder = new RegularDFABuilder<TTag>();
92 77
93 78 lang.Accept( builder );
94 79
Show file before | Show file after | Hide comments
@@ -88,16 +88,6
88 88 <Compile Include="IPromise.cs" />
89 89 <Compile Include="IServiceLocator.cs" />
90 90 <Compile Include="ITaskController.cs" />
91 <Compile Include="JSON\JSONElementContext.cs" />
92 <Compile Include="JSON\JSONElementType.cs" />
93 <Compile Include="JSON\JSONGrammar.cs" />
94 <Compile Include="JSON\JSONParser.cs" />
95 <Compile Include="JSON\JSONScanner.cs" />
96 <Compile Include="JSON\JsonTokenType.cs" />
97 <Compile Include="JSON\JSONWriter.cs" />
98 <Compile Include="JSON\JSONXmlReader.cs" />
99 <Compile Include="JSON\JSONXmlReaderOptions.cs" />
100 <Compile Include="JSON\StringTranslator.cs" />
101 91 <Compile Include="Parallels\DispatchPool.cs" />
102 92 <Compile Include="Parallels\ArrayTraits.cs" />
103 93 <Compile Include="Parallels\MTQueue.cs" />
@@ -183,9 +173,21
183 173 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
184 174 <Compile Include="Automaton\RegularExpressions\Token.cs" />
185 175 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
186 <Compile Include="Automaton\RegularExpressions\DFABuilder.cs" />
187 176 <Compile Include="Automaton\AutomatonTransition.cs" />
188 177 <Compile Include="Automaton\ByteAlphabet.cs" />
178 <Compile Include="Automaton\RegularExpressions\RegularDFABuilder.cs" />
179 <Compile Include="Formats\JSON\JSONElementContext.cs" />
180 <Compile Include="Formats\JSON\JSONElementType.cs" />
181 <Compile Include="Formats\JSON\JSONGrammar.cs" />
182 <Compile Include="Formats\JSON\JSONParser.cs" />
183 <Compile Include="Formats\JSON\JSONScanner.cs" />
184 <Compile Include="Formats\JSON\JsonTokenType.cs" />
185 <Compile Include="Formats\JSON\JSONWriter.cs" />
186 <Compile Include="Formats\JSON\JSONXmlReader.cs" />
187 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
188 <Compile Include="Formats\JSON\StringTranslator.cs" />
189 <Compile Include="Automaton\MapAlphabet.cs" />
190 <Compile Include="Automaton\DummyAlphabet.cs" />
189 191 </ItemGroup>
190 192 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
191 193 <ItemGroup />
@@ -261,5 +263,7
261 263 <ItemGroup>
262 264 <Folder Include="Components\" />
263 265 <Folder Include="Automaton\RegularExpressions\" />
266 <Folder Include="Formats\" />
267 <Folder Include="Formats\JSON\" />
264 268 </ItemGroup>
265 269 </Project> No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now