##// END OF EJS Templates
implab » ImplabNet
RSS
Rewritten JsonScanner, JsonParser, fixed naming style
cin -
r228:6fa235c5a760 v2
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,42
1 using Implab.Automaton;
2 using System;
3 using System.Collections.Generic;
4 using System.Linq;
5 using System.Runtime.CompilerServices;
6 using System.Text;
7 using System.Threading.Tasks;
8
9 namespace Implab.Formats {
10 public class CharMap : IAlphabet<char> {
11 readonly char m_min;
12 readonly char m_max;
13 readonly int[] m_map;
14
15 public CharMap(char min, int[] map) {
16 Safe.ArgumentNotNull(map, nameof(map));
17 Count = map.Max()+1;
18 m_min = min;
19 m_map = map;
20 m_max = (char)(min + map.Length);
21 }
22
23 public int Count {
24 get; private set;
25 }
26
27 public bool Contains(char symbol) {
28 return symbol >= m_min && symbol <= m_max && m_map[symbol-m_min] != AutomatonConst.UNCLASSIFIED_INPUT;
29 }
30
31 public IEnumerable<char> GetSymbols(int cls) {
32 for (var i = 0; i < m_map.Length; i++)
33 if (m_map[i] == cls)
34 yield return (char)(i + m_min);
35 }
36
37 [MethodImpl(MethodImplOptions.AggressiveInlining)]
38 public int Translate(char symbol) {
39 return symbol >= m_min && symbol <= m_max ? m_map[symbol-m_min] : AutomatonConst.UNCLASSIFIED_INPUT;
40 }
41 }
42 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,84
1 using Implab.Automaton;
2 using System;
3 using System.Collections.Generic;
4 using System.Linq;
5 using System.Text;
6 using System.Threading.Tasks;
7
8 namespace Implab.Formats {
9 public class InputScanner<TTag> {
10 readonly TTag[] m_tags;
11 readonly int m_initialState;
12 readonly int[,] m_dfa;
13 readonly CharMap m_alphabet;
14 readonly bool[] m_final;
15
16 int m_position;
17 int m_state;
18
19 public InputScanner(int[,] dfaTable, bool[] finalStates, TTag[] tags, int initialState, CharMap alphabet) {
20 Safe.ArgumentNotNull(dfaTable, nameof(dfaTable));
21 Safe.ArgumentNotNull(finalStates, nameof(finalStates));
22 Safe.ArgumentNotNull(tags, nameof(tags));
23 Safe.ArgumentNotNull(alphabet, nameof(alphabet));
24
25 m_dfa = dfaTable;
26 m_final = finalStates;
27 m_tags = tags;
28 m_initialState = initialState;
29 m_alphabet = alphabet;
30 }
31
32 public TTag Tag {
33 get {
34 return m_tags[m_state];
35 }
36 }
37
38 public int Position {
39 get {
40 return m_position;
41 }
42 }
43
44 public bool IsFinal {
45 get {
46 return m_final[m_state];
47 }
48 }
49
50 public void Reset() {
51 m_state = m_initialState;
52 }
53
54 public InputScanner<TTag> Clone() {
55 var clone = new InputScanner<TTag>(m_dfa, m_final, m_tags, m_initialState, m_alphabet);
56 clone.m_state = m_state;
57 clone.m_position = m_position;
58 return clone;
59 }
60
61 public bool Scan(char[] data, int offset, int length) {
62 if (length <= 0) {
63 m_position = offset;
64 return false; // EOF
65 }
66
67 var max = offset + length;
68 var next = m_state;
69
70 while(offset < max) {
71 next = m_dfa[next, m_alphabet.Translate(data[offset])];
72 if (next == AutomatonConst.UNREACHABLE_STATE) {
73 // scanner stops on the next position after last recognized symbol
74 m_position = offset;
75 return false;
76 }
77 m_state = next;
78 offset++;
79 }
80 m_position = offset;
81 return true;
82 }
83 }
84 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,76
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Threading.Tasks;
6
7 namespace Implab.Formats.Json {
8 public class JsonStringScanner : JsonScanner {
9 const int _defaultBuffer = 64;
10
11 readonly string m_data;
12 int m_offset;
13
14 JsonStringScanner(string data, char[] buffer, int pos, int length, int offset) : base(buffer, pos, length) {
15 m_data = data;
16 m_offset = offset;
17 }
18
19 protected override int Read(char[] buffer, int offset, int size) {
20 if (m_data == null)
21 return 0;
22 if (m_offset >= m_data.Length)
23 return 0;
24
25 var count = Math.Min(size, m_data.Length - m_offset);
26
27 m_data.CopyTo(m_offset, buffer, offset, count);
28 m_offset += count;
29
30 return count;
31 }
32
33 public static JsonStringScanner Create(string data) {
34 Safe.ArgumentNotNull(data, nameof(data));
35
36 if (data.Length <= _defaultBuffer)
37 return new JsonStringScanner(null, data.ToCharArray(), 0, data.Length, data.Length);
38
39 var buffer = new char[_defaultBuffer];
40 data.CopyTo(0, buffer, 0, _defaultBuffer);
41 return new JsonStringScanner(data, buffer, 0, _defaultBuffer, _defaultBuffer);
42 }
43
44 public static JsonStringScanner Create(string data, int offset, int length) {
45 Safe.ArgumentNotNull(data, nameof(data));
46 Safe.ArgumentGreaterThan(offset, 0, nameof(offset));
47 Safe.ArgumentGreaterThan(length, 0, nameof(length));
48
49 if (offset + length > data.Length)
50 throw new ArgumentOutOfRangeException("Specified offset and length are out of the string bounds");
51
52 if (length <= _defaultBuffer) {
53 var buffer = new char[length];
54 data.CopyTo(offset, buffer, 0, length);
55
56 return new JsonStringScanner(null, buffer, 0, length, length);
57 } else {
58 var buffer = new char[_defaultBuffer];
59 data.CopyTo(offset, buffer, 0, _defaultBuffer);
60 return new JsonStringScanner(data, buffer, 0, _defaultBuffer, offset + _defaultBuffer);
61 }
62 }
63
64 public static JsonStringScanner Create(char[] data, int offset, int length) {
65 Safe.ArgumentNotNull(data, nameof(data));
66 Safe.ArgumentGreaterThan(offset, 0, nameof(offset));
67 Safe.ArgumentGreaterThan(length, 0, nameof(length));
68
69 if (offset + length > data.Length)
70 throw new ArgumentOutOfRangeException("Specified offset and length are out of the array bounds");
71
72 return new JsonStringScanner(null, data, offset, offset + length, offset + length);
73
74 }
75 }
76 }
Show file before | Show file after | Hide comments
@@ -0,0 +1,49
1 using System;
2 using System.Collections.Generic;
3 using System.IO;
4 using System.Linq;
5 using System.Text;
6 using System.Threading.Tasks;
7
8 namespace Implab.Formats.Json {
9 public class JsonTextScanner : JsonScanner {
10 const int _bufferSize = 4096;
11 readonly TextReader m_reader;
12
13 JsonTextScanner(TextReader reader, char[] buffer) : base(buffer, 0, 0) {
14 m_reader = reader;
15 }
16
17 protected override int Read(char[] buffer, int offset, int size) {
18 return m_reader.Read(buffer, offset, size);
19 }
20
21 public static JsonTextScanner Create(string file, Encoding encoding) {
22 return new JsonTextScanner(new StreamReader(file, encoding), new char[_bufferSize]);
23 }
24
25 public static JsonTextScanner Create(string file) {
26 return new JsonTextScanner(new StreamReader(file), new char[_bufferSize]);
27 }
28
29 public static JsonTextScanner Create(Stream stream, Encoding encoding) {
30 return new JsonTextScanner(new StreamReader(stream, encoding), new char[_bufferSize]);
31 }
32
33 public static JsonTextScanner Create(Stream stream) {
34 return new JsonTextScanner(new StreamReader(stream), new char[_bufferSize]);
35 }
36
37 public static JsonTextScanner Create(TextReader reader) {
38 Safe.ArgumentNotNull(reader, nameof(reader));
39 return new JsonTextScanner(reader, new char[_bufferSize]);
40 }
41
42 protected override void Dispose(bool disposing) {
43 if (disposing)
44 Safe.Dispose(m_reader);
45
46 base.Dispose(disposing);
47 }
48 }
49 }
Show file before | Show file after | Hide comments
@@ -1,41 +1,42
1 using NUnit.Framework;
1 using NUnit.Framework;
2 using System;
2 using System;
3 using Implab.Formats.JSON;
4 using Implab.Automaton;
3 using Implab.Automaton;
5 using Implab.Xml;
4 using Implab.Xml;
6 using System.Xml;
5 using System.Xml;
7 using System.Text;
6 using Implab.Formats;
7 using Implab.Formats.Json;
8
8
9 namespace Implab.Format.Test {
9 namespace Implab.Format.Test {
10 [TestFixture]
10 [TestFixture]
11 public class JsonTests {
11 public class JsonTests {
12
12 [Test]
13 [Test]
13 public void TestScannerValidTokens() {
14 public void TestScannerValidTokens() {
14 using (var scanner = new JSONScanner(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
15 using (var scanner = JsonStringScanner.Create(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
15
16
16 Tuple<JsonTokenType, object>[] expexted = {
17 Tuple<JsonTokenType, object>[] expexted = {
17 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 9123d),
18 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 9123d),
18 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
19 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
19 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -123d),
20 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -123d),
20 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
21 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
21 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0d),
22 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0d),
22 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
23 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
23 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0.1d),
24 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 0.1d),
24 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
25 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
25 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.2d),
26 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.2d),
26 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
27 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
27 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.1e3d),
28 new Tuple<JsonTokenType,object>(JsonTokenType.Number, -0.1e3d),
28 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
29 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
29 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 1.3E-3d),
30 new Tuple<JsonTokenType,object>(JsonTokenType.Number, 1.3E-3d),
30 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
31 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
31 new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n text"),
32 new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n text"),
32 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, ", "),
33 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
33 new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal"),
34 new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal"),
34 new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, " ["),
35 new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, null),
35 new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, "]"),
36 new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, null),
36 new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, "{"),
37 new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, null),
37 new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, "}"),
38 new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, null),
38 new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, ":")
39 new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, null)
39 };
40 };
40
41
41 object value;
42 object value;
@@ -70,7 +71,7 namespace Implab.Format.Test {
70 };
71 };
71
72
72 foreach (var json in bad) {
73 foreach (var json in bad) {
73 using (var scanner = new JSONScanner(json)) {
74 using (var scanner = JsonStringScanner.Create(json)) {
74 try {
75 try {
75 object value;
76 object value;
76 JsonTokenType token;
77 JsonTokenType token;
@@ -122,7 +123,7 namespace Implab.Format.Test {
122 void DumpJsonParse(string json) {
123 void DumpJsonParse(string json) {
123 Console.WriteLine($"JSON: {json}");
124 Console.WriteLine($"JSON: {json}");
124 Console.WriteLine("XML");
125 Console.WriteLine("XML");
125 using (var xmlReader = new JsonXmlReader(new JSONParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "json" })) {
126 using (var xmlReader = new JsonXmlReader(new JsonParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "json" })) {
126 while (xmlReader.Read())
127 while (xmlReader.Read())
127 Console.WriteLine($"{new string(' ', xmlReader.Depth * 2)}{xmlReader}");
128 Console.WriteLine($"{new string(' ', xmlReader.Depth * 2)}{xmlReader}");
128 }
129 }
@@ -136,7 +137,7 namespace Implab.Format.Test {
136 CloseOutput = false,
137 CloseOutput = false,
137 ConformanceLevel = ConformanceLevel.Document
138 ConformanceLevel = ConformanceLevel.Document
138 }))
139 }))
139 using (var xmlReader = new JsonXmlReader(new JSONParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "", FlattenArrays = true })) {
140 using (var xmlReader = new JsonXmlReader(new JsonParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "", FlattenArrays = true })) {
140 xmlWriter.WriteNode(xmlReader, false);
141 xmlWriter.WriteNode(xmlReader, false);
141 }
142 }
142 }
143 }
Show file before | Show file after | Hide comments
@@ -1,6 +1,7
1 using System.Collections.Generic;
1 using System.Collections.Generic;
2 using System.Linq;
2 using System.Linq;
3 using Implab.Automaton;
3 using Implab.Automaton;
4 using System;
4
5
5 namespace Implab.Formats {
6 namespace Implab.Formats {
6 public class CharAlphabet: IndexedAlphabetBase<char> {
7 public class CharAlphabet: IndexedAlphabetBase<char> {
@@ -12,5 +13,24 namespace Implab.Formats {
12 public IEnumerable<char> InputSymbols {
13 public IEnumerable<char> InputSymbols {
13 get { return Enumerable.Range(char.MinValue, char.MaxValue).Cast<char>(); }
14 get { return Enumerable.Range(char.MinValue, char.MaxValue).Cast<char>(); }
14 }
15 }
16
17 public CharMap CreateCharMap() {
18 var map = new Dictionary<int, int>();
19
20 int max = 0, min = char.MaxValue;
21 foreach (var p in Mappings) {
22 var index = GetSymbolIndex(p.Key);
23 max = Math.Max(max, index);
24 min = Math.Min(min, index);
25 map[index] = p.Value;
26 }
27
28 var result = new int[max - min + 1];
29
30 for (int i = 0; i < result.Length; i++)
31 map.TryGetValue(min + i, out result[i]);
32
33 return new CharMap((char)min, result);
15 }
34 }
16 }
35 }
36 }
Show file before | Show file after | Hide comments
@@ -67,32 +67,6 namespace Implab.Formats {
67
67
68 return Token.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
68 return Token.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
69 }
69 }
70
71 protected abstract IndexedAlphabetBase<TSymbol> CreateAlphabet();
72
73 protected ScannerContext<TTag> BuildScannerContext<TTag>(Token regexp) {
74
75 var dfa = new RegularDFA<TSymbol, TTag>(AlphabetBuilder);
76
77 var visitor = new RegularExpressionVisitor<TTag>(dfa);
78 regexp.Accept(visitor);
79 visitor.BuildDFA();
80
81 if (dfa.IsFinalState(dfa.InitialState))
82 throw new ApplicationException("The specified language contains empty token");
83
84 var ab = CreateAlphabet();
85 var optimal = dfa.Optimize(ab);
86
87 return new ScannerContext<TTag>(
88 optimal.CreateTransitionTable(),
89 optimal.CreateFinalStateTable(),
90 optimal.CreateTagTable(),
91 optimal.InitialState,
92 ab.GetTranslationMap()
93 );
94 }
95
96 }
70 }
97
71
98
72
Show file before | Show file after | Hide comments
@@ -1,8 +1,8
1 namespace Implab.Formats.JSON {
1 namespace Implab.Formats.Json {
2 /// <summary>
2 /// <summary>
3 /// internal
3 /// internal
4 /// </summary>
4 /// </summary>
5 enum JSONElementContext {
5 enum JsonElementContext {
6 None,
6 None,
7 Object,
7 Object,
8 Array,
8 Array,
Show file before | Show file after | Hide comments
@@ -1,8 +1,8
1 namespace Implab.Formats.JSON {
1 namespace Implab.Formats.Json {
2 /// <summary>
2 /// <summary>
3 /// Тип элемента на котором находится парсер
3 /// Тип элемента на котором находится парсер
4 /// </summary>
4 /// </summary>
5 public enum JSONElementType {
5 public enum JsonElementType {
6 None,
6 None,
7 /// <summary>
7 /// <summary>
8 /// Начало объекта
8 /// Начало объекта
Show file before | Show file after | Hide comments
@@ -4,8 +4,8 using System;
4 using Implab.Automaton;
4 using Implab.Automaton;
5 using Implab.Components;
5 using Implab.Components;
6
6
7 namespace Implab.Formats.JSON {
7 namespace Implab.Formats.Json {
8 class JSONGrammar : Grammar<char> {
8 public class JsonGrammar : Grammar<char> {
9 public enum TokenType {
9 public enum TokenType {
10 None,
10 None,
11 BeginObject,
11 BeginObject,
@@ -25,17 +25,19 namespace Implab.Formats.JSON {
25 EscapedUnicode
25 EscapedUnicode
26 }
26 }
27
27
28 static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
28 static LazyAndWeak<JsonGrammar> _instance = new LazyAndWeak<JsonGrammar>(() => new JsonGrammar());
29
29
30 public static JSONGrammar Instance {
30 public static JsonGrammar Instance {
31 get { return _instance.Value; }
31 get { return _instance.Value; }
32 }
32 }
33
33
34 readonly ScannerContext<TokenType> m_jsonExpression;
34 readonly InputScanner<TokenType> m_jsonExpression;
35 readonly ScannerContext<TokenType> m_stringExpression;
35 readonly InputScanner<TokenType> m_stringExpression;
36 readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
36 readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
37
37
38 public JSONGrammar() {
38 public CharAlphabet DefaultAlphabet { get { return m_defaultAlphabet; } }
39
40 public JsonGrammar() {
39 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
41 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
40 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
42 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
41 var digit9 = SymbolRangeToken('1', '9');
43 var digit9 = SymbolRangeToken('1', '9');
@@ -85,10 +87,16 namespace Implab.Formats.JSON {
85 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
87 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
86
88
87
89
88 m_jsonExpression = BuildScannerContext<TokenType>(jsonExpression);
90 m_jsonExpression = BuildScanner(jsonExpression);
89 m_stringExpression = BuildScannerContext<TokenType>(jsonStringExpression);
91 m_stringExpression = BuildScanner(jsonStringExpression);
92 }
90
93
94 public static InputScanner<TokenType> CreateJsonExpressionScanner() {
95 return Instance.m_jsonExpression.Clone();
96 }
91
97
98 public static InputScanner<TokenType> CreateStringExpressionScanner() {
99 return Instance.m_stringExpression.Clone();
92 }
100 }
93
101
94 protected override IAlphabetBuilder<char> AlphabetBuilder {
102 protected override IAlphabetBuilder<char> AlphabetBuilder {
@@ -97,24 +105,43 namespace Implab.Formats.JSON {
97 }
105 }
98 }
106 }
99
107
100 public ScannerContext<TokenType> JsonExpression {
101 get {
102 return m_jsonExpression;
103 }
104 }
105
106 public ScannerContext<TokenType> JsonStringExpression {
107 get {
108 return m_stringExpression;
109 }
110 }
111
112 Token SymbolRangeToken(char start, char stop) {
108 Token SymbolRangeToken(char start, char stop) {
113 return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
109 return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
114 }
110 }
115
111
116 protected override IndexedAlphabetBase<char> CreateAlphabet() {
112 public InputScanner<TokenType> BuildScanner(Token regexp) {
117 return new CharAlphabet();
113 var dfa = new RegularDFA<char, TokenType>(AlphabetBuilder);
114
115 var visitor = new RegularExpressionVisitor<TokenType>(dfa);
116 regexp.Accept(visitor);
117 visitor.BuildDFA();
118
119 if (dfa.IsFinalState(dfa.InitialState))
120 throw new ApplicationException("The specified language contains empty token");
121
122 var ab = new CharAlphabet();
123 var optimal = dfa.Optimize(ab);
124
125 return new InputScanner<TokenType>(
126 optimal.CreateTransitionTable(),
127 optimal.CreateFinalStateTable(),
128 NormalizeTags(optimal.CreateTagTable()),
129 optimal.InitialState,
130 ab.CreateCharMap()
131 );
132 }
133
134 static TokenType[] NormalizeTags(TokenType[][] tags) {
135 var result = new TokenType[tags.Length];
136 for(var i = 0; i< tags.Length; i++) {
137 if (tags[i] == null || tags[i].Length == 0)
138 result[i] = default(TokenType);
139 else if (tags[i].Length == 1)
140 result[i] = tags[i][0];
141 else
142 throw new Exception($"Ambigous state tags {string.Join(", ", tags[i])}");
143 }
144 return result;
118 }
145 }
119
146
120 }
147 }
Show file before | Show file after | Hide comments
@@ -7,7 +7,7 using System.Linq;
7 using Implab.Components;
7 using Implab.Components;
8 using System.Collections.Generic;
8 using System.Collections.Generic;
9
9
10 namespace Implab.Formats.JSON {
10 namespace Implab.Formats.Json {
11 /// <summary>
11 /// <summary>
12 /// Pull парсер JSON данных.
12 /// Pull парсер JSON данных.
13 /// </summary>
13 /// </summary>
@@ -24,7 +24,7 namespace Implab.Formats.JSON {
24 /// } // Level = 0
24 /// } // Level = 0
25 /// </code>
25 /// </code>
26 /// </remarks>
26 /// </remarks>
27 public class JSONParser : Disposable {
27 public class JsonParser : Disposable {
28
28
29 enum MemberContext {
29 enum MemberContext {
30 MemberName,
30 MemberName,
@@ -36,9 +36,9 namespace Implab.Formats.JSON {
36 readonly int[,] m_dfa;
36 readonly int[,] m_dfa;
37 int m_state;
37 int m_state;
38
38
39 readonly JSONElementContext m_elementContext;
39 readonly JsonElementContext m_elementContext;
40
40
41 public ParserContext(int[,] dfa, int state, JSONElementContext context) {
41 public ParserContext(int[,] dfa, int state, JsonElementContext context) {
42 m_dfa = dfa;
42 m_dfa = dfa;
43 m_state = state;
43 m_state = state;
44 m_elementContext = context;
44 m_elementContext = context;
@@ -52,7 +52,7 namespace Implab.Formats.JSON {
52 return true;
52 return true;
53 }
53 }
54
54
55 public JSONElementContext ElementContext {
55 public JsonElementContext ElementContext {
56 get { return m_elementContext; }
56 get { return m_elementContext; }
57 }
57 }
58 }
58 }
@@ -61,7 +61,7 namespace Implab.Formats.JSON {
61 static readonly ParserContext _objectContext;
61 static readonly ParserContext _objectContext;
62 static readonly ParserContext _arrayContext;
62 static readonly ParserContext _arrayContext;
63
63
64 static JSONParser() {
64 static JsonParser() {
65
65
66 var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
66 var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
67 var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
67 var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
@@ -88,16 +88,16 namespace Implab.Formats.JSON {
88
88
89 var jsonExpression = valueExpression.End();
89 var jsonExpression = valueExpression.End();
90
90
91 _jsonContext = CreateParserContext(jsonExpression, JSONElementContext.None);
91 _jsonContext = CreateParserContext(jsonExpression, JsonElementContext.None);
92 _objectContext = CreateParserContext(objectExpression, JSONElementContext.Object);
92 _objectContext = CreateParserContext(objectExpression, JsonElementContext.Object);
93 _arrayContext = CreateParserContext(arrayExpression, JSONElementContext.Array);
93 _arrayContext = CreateParserContext(arrayExpression, JsonElementContext.Array);
94 }
94 }
95
95
96 static Token MakeToken(params JsonTokenType[] input) {
96 static Token MakeToken(params JsonTokenType[] input) {
97 return Token.New( input.Select(t => (int)t).ToArray() );
97 return Token.New( input.Select(t => (int)t).ToArray() );
98 }
98 }
99
99
100 static ParserContext CreateParserContext(Token expr, JSONElementContext context) {
100 static ParserContext CreateParserContext(Token expr, JsonElementContext context) {
101
101
102 var dfa = new DFATable();
102 var dfa = new DFATable();
103 var builder = new RegularExpressionVisitor(dfa);
103 var builder = new RegularExpressionVisitor(dfa);
@@ -109,11 +109,11 namespace Implab.Formats.JSON {
109
109
110 #endregion
110 #endregion
111
111
112 readonly JSONScanner m_scanner;
112 readonly JsonScanner m_scanner;
113 // json starts from the value context and may content even a single literal
113 // json starts from the value context and may content even a single literal
114 MemberContext m_memberContext = MemberContext.MemberValue;
114 MemberContext m_memberContext = MemberContext.MemberValue;
115
115
116 JSONElementType m_elementType;
116 JsonElementType m_elementType;
117 object m_elementValue;
117 object m_elementValue;
118 string m_memberName = String.Empty;
118 string m_memberName = String.Empty;
119
119
@@ -124,18 +124,18 namespace Implab.Formats.JSON {
124 /// Создает новый парсер на основе строки, содержащей JSON
124 /// Создает новый парсер на основе строки, содержащей JSON
125 /// </summary>
125 /// </summary>
126 /// <param name="text"></param>
126 /// <param name="text"></param>
127 public JSONParser(string text) {
127 public JsonParser(string text) {
128 Safe.ArgumentNotEmpty(text, "text");
128 Safe.ArgumentNotEmpty(text, "text");
129 m_scanner = new JSONScanner(text);
129 m_scanner = JsonStringScanner.Create(text);
130 }
130 }
131
131
132 /// <summary>
132 /// <summary>
133 /// Создает новый экземпляр парсера, на основе текстового потока.
133 /// Создает новый экземпляр парсера, на основе текстового потока.
134 /// </summary>
134 /// </summary>
135 /// <param name="reader">Текстовый поток.</param>
135 /// <param name="reader">Текстовый поток.</param>
136 public JSONParser(TextReader reader) {
136 public JsonParser(TextReader reader) {
137 Safe.ArgumentNotNull(reader, "reader");
137 Safe.ArgumentNotNull(reader, "reader");
138 m_scanner = new JSONScanner(reader);
138 m_scanner = JsonTextScanner.Create(reader);
139 }
139 }
140
140
141 public int Level {
141 public int Level {
@@ -145,7 +145,7 namespace Implab.Formats.JSON {
145 /// <summary>
145 /// <summary>
146 /// Тип текущего элемента на котором стоит парсер.
146 /// Тип текущего элемента на котором стоит парсер.
147 /// </summary>
147 /// </summary>
148 public JSONElementType ElementType {
148 public JsonElementType ElementType {
149 get { return m_elementType; }
149 get { return m_elementType; }
150 }
150 }
151
151
@@ -158,7 +158,7 namespace Implab.Formats.JSON {
158 }
158 }
159
159
160 /// <summary>
160 /// <summary>
161 /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c>
161 /// Значение элемента. Только для элементов типа <see cref="JsonElementType.Value"/>, для остальных <c>null</c>
162 /// </summary>
162 /// </summary>
163 public object ElementValue {
163 public object ElementValue {
164 get { return m_elementValue; }
164 get { return m_elementValue; }
@@ -185,7 +185,7 namespace Implab.Formats.JSON {
185
185
186 m_elementValue = null;
186 m_elementValue = null;
187 m_memberContext = MemberContext.MemberName;
187 m_memberContext = MemberContext.MemberName;
188 m_elementType = JSONElementType.BeginObject;
188 m_elementType = JsonElementType.BeginObject;
189 return true;
189 return true;
190 case JsonTokenType.EndObject:
190 case JsonTokenType.EndObject:
191 if (m_stack.Count == 0)
191 if (m_stack.Count == 0)
@@ -193,7 +193,7 namespace Implab.Formats.JSON {
193 m_context = m_stack.Pop();
193 m_context = m_stack.Pop();
194
194
195 m_elementValue = null;
195 m_elementValue = null;
196 m_elementType = JSONElementType.EndObject;
196 m_elementType = JsonElementType.EndObject;
197 return true;
197 return true;
198 case JsonTokenType.BeginArray:
198 case JsonTokenType.BeginArray:
199 m_stack.Push(m_context);
199 m_stack.Push(m_context);
@@ -201,7 +201,7 namespace Implab.Formats.JSON {
201
201
202 m_elementValue = null;
202 m_elementValue = null;
203 m_memberContext = MemberContext.MemberValue;
203 m_memberContext = MemberContext.MemberValue;
204 m_elementType = JSONElementType.BeginArray;
204 m_elementType = JsonElementType.BeginArray;
205 return true;
205 return true;
206 case JsonTokenType.EndArray:
206 case JsonTokenType.EndArray:
207 if (m_stack.Count == 0)
207 if (m_stack.Count == 0)
@@ -209,36 +209,36 namespace Implab.Formats.JSON {
209 m_context = m_stack.Pop();
209 m_context = m_stack.Pop();
210
210
211 m_elementValue = null;
211 m_elementValue = null;
212 m_elementType = JSONElementType.EndArray;
212 m_elementType = JsonElementType.EndArray;
213 return true;
213 return true;
214 case JsonTokenType.String:
214 case JsonTokenType.String:
215 if (m_memberContext == MemberContext.MemberName) {
215 if (m_memberContext == MemberContext.MemberName) {
216 m_memberName = (string)tokenValue;
216 m_memberName = (string)tokenValue;
217 break;
217 break;
218 }
218 }
219 m_elementType = JSONElementType.Value;
219 m_elementType = JsonElementType.Value;
220 m_elementValue = tokenValue;
220 m_elementValue = tokenValue;
221 return true;
221 return true;
222 case JsonTokenType.Number:
222 case JsonTokenType.Number:
223 m_elementType = JSONElementType.Value;
223 m_elementType = JsonElementType.Value;
224 m_elementValue = tokenValue;
224 m_elementValue = tokenValue;
225 return true;
225 return true;
226 case JsonTokenType.Literal:
226 case JsonTokenType.Literal:
227 m_elementType = JSONElementType.Value;
227 m_elementType = JsonElementType.Value;
228 m_elementValue = ParseLiteral((string)tokenValue);
228 m_elementValue = ParseLiteral((string)tokenValue);
229 return true;
229 return true;
230 case JsonTokenType.NameSeparator:
230 case JsonTokenType.NameSeparator:
231 m_memberContext = MemberContext.MemberValue;
231 m_memberContext = MemberContext.MemberValue;
232 break;
232 break;
233 case JsonTokenType.ValueSeparator:
233 case JsonTokenType.ValueSeparator:
234 m_memberContext = m_context.ElementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
234 m_memberContext = m_context.ElementContext == JsonElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
235 break;
235 break;
236 default:
236 default:
237 UnexpectedToken(tokenValue, tokenType);
237 UnexpectedToken(tokenValue, tokenType);
238 break;
238 break;
239 }
239 }
240 }
240 }
241 if (m_context.ElementContext != JSONElementContext.None)
241 if (m_context.ElementContext != JsonElementContext.None)
242 throw new ParserException("Unexpedted end of data");
242 throw new ParserException("Unexpedted end of data");
243
243
244 EOF = true;
244 EOF = true;
Show file before | Show file after | Hide comments
@@ -5,37 +5,65 using System.Text;
5 using Implab.Components;
5 using Implab.Components;
6 using System.IO;
6 using System.IO;
7
7
8 namespace Implab.Formats.JSON {
8 namespace Implab.Formats.Json {
9 /// <summary>
9 /// <summary>
10 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
10 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
11 /// </summary>
11 /// </summary>
12 public class JSONScanner : Disposable {
12 public abstract class JsonScanner : Disposable {
13 readonly StringBuilder m_builder = new StringBuilder();
13 readonly InputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
14
14 readonly InputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
15 readonly ScannerContext<JSONGrammar.TokenType> m_jsonContext = JSONGrammar.Instance.JsonExpression;
16 readonly ScannerContext<JSONGrammar.TokenType> m_stringContext = JSONGrammar.Instance.JsonStringExpression;
17
18
15
19 readonly TextScanner m_scanner;
16 readonly char[] m_unescapeBuf = new char[4];
17 readonly char[] m_buffer;
18 int m_length;
19 int m_pos;
20 readonly StringBuilder m_tokenBuilder = new StringBuilder();
20
21
21 /// <summary>
22 protected JsonScanner(char[] buffer, int pos, int length) {
22 /// Создает новый экземпляр сканнера
23 m_buffer = buffer;
23 /// </summary>
24 m_pos = pos;
24 public JSONScanner(string text) {
25 m_length = length;
25 Safe.ArgumentNotEmpty(text, "text");
26
27 m_scanner = new StringScanner(text);
28 }
26 }
29
27
30 public JSONScanner(TextReader reader, int bufferMax, int chunkSize) {
28 bool Read(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
31 Safe.ArgumentNotNull(reader, "reader");
29 scanner.Reset();
32
30
33 m_scanner = new ReaderScanner(reader, bufferMax, chunkSize);
31 if (m_pos == m_length) {
32 m_pos = 0;
33 m_length = Read(m_buffer, 0, m_buffer.Length);
34 if (m_length == 0) {
35 tokenType = JsonGrammar.TokenType.None;
36 return false; // EOF
37 }
34 }
38 }
35
39
36 public JSONScanner(TextReader reader) : this(reader, 1024*1024, 1024){
40 while(scanner.Scan(m_buffer, m_pos, m_length - m_pos)) {
41 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
42 m_pos = 0;
43 m_length = Read(m_buffer, 0, m_buffer.Length);
44 }
45 var scannerPos = scanner.Position;
46 if (scannerPos != m_pos) {
47 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
48 m_pos = scannerPos;
37 }
49 }
38
50
51 if (!scanner.IsFinal) {
52 if (m_length == 0) {
53 // unexpected EOF
54 throw new ParserException("Unexpected EOF");
55 } else {
56 // unecpected character
57 throw new ParserException($"Unexpected character '{m_buffer[m_pos + 1]}'");
58 }
59 }
60 tokenType = scanner.Tag;
61 return true;
62 }
63
64 protected abstract int Read(char[] buffer, int offset, int size);
65
66
39 /// <summary>
67 /// <summary>
40 /// Читает следующий лексический элемент из входных данных.
68 /// Читает следующий лексический элемент из входных данных.
41 /// </summary>
69 /// </summary>
@@ -45,22 +73,28 namespace Implab.Formats.JSON {
45 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
73 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
46 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
74 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
47 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
75 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
48 JSONGrammar.TokenType[] tag;
76 JsonGrammar.TokenType tag;
49 while (m_jsonContext.Execute(m_scanner, out tag)) {
77 m_tokenBuilder.Clear();
50 switch (tag[0]) {
78 while (Read(m_jsonContext, out tag)) {
51 case JSONGrammar.TokenType.StringBound:
79 switch (tag) {
80 case JsonGrammar.TokenType.StringBound:
52 tokenValue = ReadString();
81 tokenValue = ReadString();
53 tokenType = JsonTokenType.String;
82 tokenType = JsonTokenType.String;
54 break;
83 break;
55 case JSONGrammar.TokenType.Number:
84 case JsonGrammar.TokenType.Number:
56 tokenValue = Double.Parse(m_scanner.GetTokenValue(), CultureInfo.InvariantCulture);
85 tokenValue = Double.Parse(m_tokenBuilder.ToString(), CultureInfo.InvariantCulture);
57 tokenType = JsonTokenType.Number;
86 tokenType = JsonTokenType.Number;
58 break;
87 break;
59 case JSONGrammar.TokenType.Whitespace:
88 case JsonGrammar.TokenType.Literal:
89 tokenType = JsonTokenType.Literal;
90 tokenValue = m_tokenBuilder.ToString();
91 break;
92 case JsonGrammar.TokenType.Whitespace:
93 m_tokenBuilder.Clear();
60 continue;
94 continue;
61 default:
95 default:
62 tokenType = (JsonTokenType)tag[0];
96 tokenType = (JsonTokenType)tag;
63 tokenValue = m_scanner.GetTokenValue();
97 tokenValue = null;
64 break;
98 break;
65 }
99 }
66 return true;
100 return true;
@@ -71,39 +105,30 namespace Implab.Formats.JSON {
71 }
105 }
72
106
73 string ReadString() {
107 string ReadString() {
74 int pos = 0;
108 JsonGrammar.TokenType tag;
75 var buf = new char[6]; // the buffer for unescaping chars
109 m_tokenBuilder.Clear();
76
77 JSONGrammar.TokenType[] tag;
78 m_builder.Clear();
79
110
80 while (m_stringContext.Execute(m_scanner, out tag)) {
111 while (Read(m_stringContext, out tag)) {
81 switch (tag[0]) {
112 switch (tag) {
82 case JSONGrammar.TokenType.StringBound:
113 case JsonGrammar.TokenType.StringBound:
83 return m_builder.ToString();
114 m_tokenBuilder.Length--;
84 case JSONGrammar.TokenType.UnescapedChar:
115 return m_tokenBuilder.ToString();
85 m_scanner.CopyTokenTo(m_builder);
116 case JsonGrammar.TokenType.UnescapedChar:
86 break;
117 break;
87 case JSONGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
118 case JsonGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
88 m_scanner.CopyTokenTo(buf, 0);
119 m_tokenBuilder.CopyTo(m_tokenBuilder.Length - 4, m_unescapeBuf, 0, 4);
89 m_builder.Append(StringTranslator.TranslateHexUnicode(buf, 2));
120 m_tokenBuilder.Length -= 6;
90 pos++;
121 m_tokenBuilder.Append(StringTranslator.TranslateHexUnicode(m_unescapeBuf, 0));
91 break;
122 break;
92 case JSONGrammar.TokenType.EscapedChar: // \t - escape sequence
123 case JsonGrammar.TokenType.EscapedChar: // \t - escape sequence
93 m_scanner.CopyTokenTo(buf, 0);
124 var ch = m_tokenBuilder[m_tokenBuilder.Length-1];
94 m_builder.Append(StringTranslator.TranslateEscapedChar(buf[1]));
125 m_tokenBuilder.Length -= 2;
126 m_tokenBuilder.Append(StringTranslator.TranslateEscapedChar(ch));
95 break;
127 break;
96 }
128 }
97
98 }
129 }
99
130
100 throw new ParserException("Unexpected end of data");
131 throw new ParserException("Unexpected end of data");
101 }
132 }
102
103 protected override void Dispose(bool disposing) {
104 if (disposing)
105 m_scanner.Dispose();
106 base.Dispose(disposing);
107 }
133 }
108 }
134 }
109 }
Show file before | Show file after | Hide comments
@@ -1,6 +1,6
1 namespace Implab.Formats.JSON {
1 namespace Implab.Formats.Json {
2 /// <summary>
2 /// <summary>
3 /// Тип токенов, возвращаемых <see cref="JSONScanner"/>.
3 /// Тип токенов, возвращаемых <see cref="JsonScanner"/>.
4 /// </summary>
4 /// </summary>
5 public enum JsonTokenType : int {
5 public enum JsonTokenType : int {
6 None = 0,
6 None = 0,
Show file before | Show file after | Hide comments
@@ -4,11 +4,11 using System.IO;
4 using System.Globalization;
4 using System.Globalization;
5 using System.Diagnostics;
5 using System.Diagnostics;
6
6
7 namespace Implab.Formats.JSON {
7 namespace Implab.Formats.Json {
8 public class JSONWriter {
8 public class JsonWriter {
9 struct Context {
9 struct Context {
10 public bool needComma;
10 public bool needComma;
11 public JSONElementContext element;
11 public JsonElementContext element;
12 }
12 }
13 Stack<Context> m_contextStack = new Stack<Context>();
13 Stack<Context> m_contextStack = new Stack<Context>();
14 Context m_context;
14 Context m_context;
@@ -30,7 +30,7 namespace Implab.Formats.JSON {
30 _escapeBSLASH,
30 _escapeBSLASH,
31 _escapeQ;
31 _escapeQ;
32
32
33 static JSONWriter() {
33 static JsonWriter() {
34 _escapeBKS = "\\b".ToCharArray();
34 _escapeBKS = "\\b".ToCharArray();
35 _escapeFWD = "\\f".ToCharArray();
35 _escapeFWD = "\\f".ToCharArray();
36 _escapeCR = "\\r".ToCharArray();
36 _escapeCR = "\\r".ToCharArray();
@@ -40,12 +40,12 namespace Implab.Formats.JSON {
40 _escapeQ = "\\\"".ToCharArray();
40 _escapeQ = "\\\"".ToCharArray();
41 }
41 }
42
42
43 public JSONWriter(TextWriter writer) {
43 public JsonWriter(TextWriter writer) {
44 Safe.ArgumentNotNull(writer, "writer");
44 Safe.ArgumentNotNull(writer, "writer");
45 m_writer = writer;
45 m_writer = writer;
46 }
46 }
47
47
48 public JSONWriter(TextWriter writer, bool indent) {
48 public JsonWriter(TextWriter writer, bool indent) {
49 Safe.ArgumentNotNull(writer, "writer");
49 Safe.ArgumentNotNull(writer, "writer");
50
50
51 m_writer = writer;
51 m_writer = writer;
@@ -66,7 +66,7 namespace Implab.Formats.JSON {
66
66
67 void WriteMemberName(string name) {
67 void WriteMemberName(string name) {
68 Safe.ArgumentNotEmpty(name, "name");
68 Safe.ArgumentNotEmpty(name, "name");
69 if (m_context.element != JSONElementContext.Object)
69 if (m_context.element != JsonElementContext.Object)
70 OperationNotApplicable("WriteMember");
70 OperationNotApplicable("WriteMember");
71 if (m_context.needComma)
71 if (m_context.needComma)
72 m_writer.Write(",");
72 m_writer.Write(",");
@@ -93,7 +93,7 namespace Implab.Formats.JSON {
93 }
93 }
94
94
95 public void WriteValue(string value) {
95 public void WriteValue(string value) {
96 if (m_context.element == JSONElementContext.Array) {
96 if (m_context.element == JsonElementContext.Array) {
97
97
98 if (m_context.needComma)
98 if (m_context.needComma)
99 m_writer.Write(",");
99 m_writer.Write(",");
@@ -101,16 +101,16 namespace Implab.Formats.JSON {
101 m_context.needComma = true;
101 m_context.needComma = true;
102
102
103 Write(value);
103 Write(value);
104 } else if (m_context.element == JSONElementContext.None) {
104 } else if (m_context.element == JsonElementContext.None) {
105 Write(value);
105 Write(value);
106 m_context.element = JSONElementContext.Closed;
106 m_context.element = JsonElementContext.Closed;
107 } else {
107 } else {
108 OperationNotApplicable("WriteValue");
108 OperationNotApplicable("WriteValue");
109 }
109 }
110 }
110 }
111
111
112 public void WriteValue(bool value) {
112 public void WriteValue(bool value) {
113 if (m_context.element == JSONElementContext.Array) {
113 if (m_context.element == JsonElementContext.Array) {
114
114
115 if (m_context.needComma)
115 if (m_context.needComma)
116 m_writer.Write(",");
116 m_writer.Write(",");
@@ -118,16 +118,16 namespace Implab.Formats.JSON {
118 m_context.needComma = true;
118 m_context.needComma = true;
119
119
120 Write(value);
120 Write(value);
121 } else if (m_context.element == JSONElementContext.None) {
121 } else if (m_context.element == JsonElementContext.None) {
122 Write(value);
122 Write(value);
123 m_context.element = JSONElementContext.Closed;
123 m_context.element = JsonElementContext.Closed;
124 } else {
124 } else {
125 OperationNotApplicable("WriteValue");
125 OperationNotApplicable("WriteValue");
126 }
126 }
127 }
127 }
128
128
129 public void WriteValue(double value) {
129 public void WriteValue(double value) {
130 if (m_context.element == JSONElementContext.Array) {
130 if (m_context.element == JsonElementContext.Array) {
131
131
132 if (m_context.needComma)
132 if (m_context.needComma)
133 m_writer.Write(",");
133 m_writer.Write(",");
@@ -135,16 +135,16 namespace Implab.Formats.JSON {
135 m_context.needComma = true;
135 m_context.needComma = true;
136
136
137 Write(value);
137 Write(value);
138 } else if (m_context.element == JSONElementContext.None) {
138 } else if (m_context.element == JsonElementContext.None) {
139 Write(value);
139 Write(value);
140 m_context.element = JSONElementContext.Closed;
140 m_context.element = JsonElementContext.Closed;
141 } else {
141 } else {
142 OperationNotApplicable("WriteValue");
142 OperationNotApplicable("WriteValue");
143 }
143 }
144 }
144 }
145
145
146 public void BeginObject() {
146 public void BeginObject() {
147 if (m_context.element != JSONElementContext.None && m_context.element != JSONElementContext.Array)
147 if (m_context.element != JsonElementContext.None && m_context.element != JsonElementContext.Array)
148 OperationNotApplicable("BeginObject");
148 OperationNotApplicable("BeginObject");
149 if (m_context.needComma)
149 if (m_context.needComma)
150 m_writer.Write(",");
150 m_writer.Write(",");
@@ -155,7 +155,7 namespace Implab.Formats.JSON {
155
155
156 m_contextStack.Push(m_context);
156 m_contextStack.Push(m_context);
157
157
158 m_context = new Context { element = JSONElementContext.Object, needComma = false };
158 m_context = new Context { element = JsonElementContext.Object, needComma = false };
159 m_writer.Write("{");
159 m_writer.Write("{");
160 }
160 }
161
161
@@ -164,23 +164,23 namespace Implab.Formats.JSON {
164
164
165 m_contextStack.Push(m_context);
165 m_contextStack.Push(m_context);
166
166
167 m_context = new Context { element = JSONElementContext.Object, needComma = false };
167 m_context = new Context { element = JsonElementContext.Object, needComma = false };
168 m_writer.Write("{");
168 m_writer.Write("{");
169 }
169 }
170
170
171 public void EndObject() {
171 public void EndObject() {
172 if (m_context.element != JSONElementContext.Object)
172 if (m_context.element != JsonElementContext.Object)
173 OperationNotApplicable("EndObject");
173 OperationNotApplicable("EndObject");
174
174
175 m_context = m_contextStack.Pop();
175 m_context = m_contextStack.Pop();
176 if (m_contextStack.Count == 0)
176 if (m_contextStack.Count == 0)
177 m_context.element = JSONElementContext.Closed;
177 m_context.element = JsonElementContext.Closed;
178 WriteIndent();
178 WriteIndent();
179 m_writer.Write("}");
179 m_writer.Write("}");
180 }
180 }
181
181
182 public void BeginArray() {
182 public void BeginArray() {
183 if (m_context.element != JSONElementContext.None && m_context.element != JSONElementContext.Array)
183 if (m_context.element != JsonElementContext.None && m_context.element != JsonElementContext.Array)
184 throw new InvalidOperationException();
184 throw new InvalidOperationException();
185 if (m_context.needComma) {
185 if (m_context.needComma) {
186 m_writer.Write(",");
186 m_writer.Write(",");
@@ -190,7 +190,7 namespace Implab.Formats.JSON {
190
190
191 WriteIndent();
191 WriteIndent();
192 m_contextStack.Push(m_context);
192 m_contextStack.Push(m_context);
193 m_context = new Context { element = JSONElementContext.Array, needComma = false };
193 m_context = new Context { element = JsonElementContext.Array, needComma = false };
194 m_writer.Write("[");
194 m_writer.Write("[");
195 }
195 }
196
196
@@ -199,17 +199,17 namespace Implab.Formats.JSON {
199
199
200 m_contextStack.Push(m_context);
200 m_contextStack.Push(m_context);
201
201
202 m_context = new Context { element = JSONElementContext.Array, needComma = false };
202 m_context = new Context { element = JsonElementContext.Array, needComma = false };
203 m_writer.Write("[");
203 m_writer.Write("[");
204 }
204 }
205
205
206 public void EndArray() {
206 public void EndArray() {
207 if (m_context.element != JSONElementContext.Array)
207 if (m_context.element != JsonElementContext.Array)
208 OperationNotApplicable("EndArray");
208 OperationNotApplicable("EndArray");
209
209
210 m_context = m_contextStack.Pop();
210 m_context = m_contextStack.Pop();
211 if (m_contextStack.Count == 0)
211 if (m_contextStack.Count == 0)
212 m_context.element = JSONElementContext.Closed;
212 m_context.element = JsonElementContext.Closed;
213 WriteIndent();
213 WriteIndent();
214 m_writer.Write("]");
214 m_writer.Write("]");
215 }
215 }
Show file before | Show file after | Hide comments
@@ -7,7 +7,7 using System.Linq;
7 using System.Text;
7 using System.Text;
8 using System.Threading.Tasks;
8 using System.Threading.Tasks;
9
9
10 namespace Implab.Formats.JSON {
10 namespace Implab.Formats.Json {
11 /// <summary>
11 /// <summary>
12 /// Класс для преобразования экранированной строки JSON
12 /// Класс для преобразования экранированной строки JSON
13 /// </summary>
13 /// </summary>
Show file before | Show file after | Hide comments
@@ -83,6 +83,10
83 <Compile Include="Diagnostics\TraceEvent.cs" />
83 <Compile Include="Diagnostics\TraceEvent.cs" />
84 <Compile Include="Diagnostics\TraceEventType.cs" />
84 <Compile Include="Diagnostics\TraceEventType.cs" />
85 <Compile Include="Diagnostics\TraceSourceAttribute.cs" />
85 <Compile Include="Diagnostics\TraceSourceAttribute.cs" />
86 <Compile Include="Formats\CharMap.cs" />
87 <Compile Include="Formats\InputScanner.cs" />
88 <Compile Include="Formats\Json\JsonStringScanner.cs" />
89 <Compile Include="Formats\Json\JsonTextScanner.cs" />
86 <Compile Include="ICancellable.cs" />
90 <Compile Include="ICancellable.cs" />
87 <Compile Include="IProgressHandler.cs" />
91 <Compile Include="IProgressHandler.cs" />
88 <Compile Include="IProgressNotifier.cs" />
92 <Compile Include="IProgressNotifier.cs" />
@@ -164,16 +168,14
164 <Compile Include="Automaton\RegularExpressions\Token.cs" />
168 <Compile Include="Automaton\RegularExpressions\Token.cs" />
165 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
169 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
166 <Compile Include="Automaton\AutomatonTransition.cs" />
170 <Compile Include="Automaton\AutomatonTransition.cs" />
167 <Compile Include="Formats\JSON\JSONElementContext.cs" />
171 <Compile Include="Formats\Json\JsonElementContext.cs" />
168 <Compile Include="Formats\JSON\JSONElementType.cs" />
172 <Compile Include="Formats\Json\JsonElementType.cs" />
169 <Compile Include="Formats\JSON\JSONGrammar.cs" />
173 <Compile Include="Formats\Json\JsonGrammar.cs" />
170 <Compile Include="Formats\JSON\JSONParser.cs" />
174 <Compile Include="Formats\Json\JsonParser.cs" />
171 <Compile Include="Formats\JSON\JSONScanner.cs" />
175 <Compile Include="Formats\Json\JsonScanner.cs" />
172 <Compile Include="Formats\JSON\JsonTokenType.cs" />
176 <Compile Include="Formats\Json\JsonTokenType.cs" />
173 <Compile Include="Formats\JSON\JSONWriter.cs" />
177 <Compile Include="Formats\Json\JsonWriter.cs" />
174 <Compile Include="Formats\JSON\JSONXmlReader.cs" />
178 <Compile Include="Formats\Json\StringTranslator.cs" />
175 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
176 <Compile Include="Formats\JSON\StringTranslator.cs" />
177 <Compile Include="Automaton\MapAlphabet.cs" />
179 <Compile Include="Automaton\MapAlphabet.cs" />
178 <Compile Include="Formats\CharAlphabet.cs" />
180 <Compile Include="Formats\CharAlphabet.cs" />
179 <Compile Include="Formats\ByteAlphabet.cs" />
181 <Compile Include="Formats\ByteAlphabet.cs" />
@@ -182,10 +184,6
182 <Compile Include="Automaton\DFATable.cs" />
184 <Compile Include="Automaton\DFATable.cs" />
183 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
185 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
184 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
186 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
185 <Compile Include="Formats\TextScanner.cs" />
186 <Compile Include="Formats\StringScanner.cs" />
187 <Compile Include="Formats\ReaderScanner.cs" />
188 <Compile Include="Formats\ScannerContext.cs" />
189 <Compile Include="Formats\Grammar.cs" />
187 <Compile Include="Formats\Grammar.cs" />
190 <Compile Include="Automaton\RegularExpressions\EndTokenT.cs" />
188 <Compile Include="Automaton\RegularExpressions\EndTokenT.cs" />
191 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
189 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
Show file before | Show file after | Hide comments
@@ -5,6 +5,7 using System.Text;
5 using System.Text.RegularExpressions;
5 using System.Text.RegularExpressions;
6 using System.Diagnostics;
6 using System.Diagnostics;
7 using System.Collections;
7 using System.Collections;
8 using System.Runtime.CompilerServices;
8
9
9 #if NET_4_5
10 #if NET_4_5
10 using System.Threading.Tasks;
11 using System.Threading.Tasks;
@@ -14,11 +15,13 namespace Implab
14 {
15 {
15 public static class Safe
16 public static class Safe
16 {
17 {
18 [MethodImpl(MethodImplOptions.AggressiveInlining)]
17 public static void ArgumentAssert(bool condition, string paramName) {
19 public static void ArgumentAssert(bool condition, string paramName) {
18 if (!condition)
20 if (!condition)
19 throw new ArgumentException("The parameter is invalid", paramName);
21 throw new ArgumentException("The parameter is invalid", paramName);
20 }
22 }
21
23
24 [MethodImpl(MethodImplOptions.AggressiveInlining)]
22 public static void ArgumentMatch(string value, string paramName, Regex rx) {
25 public static void ArgumentMatch(string value, string paramName, Regex rx) {
23 if (rx == null)
26 if (rx == null)
24 throw new ArgumentNullException("rx");
27 throw new ArgumentNullException("rx");
@@ -26,26 +29,37 namespace Implab
26 throw new ArgumentException(String.Format("The prameter value must match {0}", rx), paramName);
29 throw new ArgumentException(String.Format("The prameter value must match {0}", rx), paramName);
27 }
30 }
28
31
32 [MethodImpl(MethodImplOptions.AggressiveInlining)]
29 public static void ArgumentNotEmpty(string value, string paramName) {
33 public static void ArgumentNotEmpty(string value, string paramName) {
30 if (String.IsNullOrEmpty(value))
34 if (String.IsNullOrEmpty(value))
31 throw new ArgumentException("The parameter can't be empty", paramName);
35 throw new ArgumentException("The parameter can't be empty", paramName);
32 }
36 }
33
37
38 [MethodImpl(MethodImplOptions.AggressiveInlining)]
34 public static void ArgumentNotEmpty<T>(T[] value, string paramName) {
39 public static void ArgumentNotEmpty<T>(T[] value, string paramName) {
35 if (value == null || value.Length == 0)
40 if (value == null || value.Length == 0)
36 throw new ArgumentException("The array must be not emty", paramName);
41 throw new ArgumentException("The array must be not emty", paramName);
37 }
42 }
38
43
44 [MethodImpl(MethodImplOptions.AggressiveInlining)]
39 public static void ArgumentNotNull(object value, string paramName) {
45 public static void ArgumentNotNull(object value, string paramName) {
40 if (value == null)
46 if (value == null)
41 throw new ArgumentNullException(paramName);
47 throw new ArgumentNullException(paramName);
42 }
48 }
43
49
50 [MethodImpl(MethodImplOptions.AggressiveInlining)]
51 internal static void ArgumentGreaterThan(int value, int min, string paramName) {
52 if (value < min)
53 throw new ArgumentOutOfRangeException(paramName);
54 }
55
56 [MethodImpl(MethodImplOptions.AggressiveInlining)]
44 public static void ArgumentInRange(int value, int min, int max, string paramName) {
57 public static void ArgumentInRange(int value, int min, int max, string paramName) {
45 if (value < min || value > max)
58 if (value < min || value > max)
46 throw new ArgumentOutOfRangeException(paramName);
59 throw new ArgumentOutOfRangeException(paramName);
47 }
60 }
48
61
62 [MethodImpl(MethodImplOptions.AggressiveInlining)]
49 public static void ArgumentOfType(object value, Type type, string paramName) {
63 public static void ArgumentOfType(object value, Type type, string paramName) {
50 if (!type.IsInstanceOfType(value))
64 if (!type.IsInstanceOfType(value))
51 throw new ArgumentException(String.Format("The parameter must be of type {0}", type), paramName);
65 throw new ArgumentException(String.Format("The parameter must be of type {0}", type), paramName);
Show file before | Show file after | Hide comments
@@ -1,10 +1,8
1 using Implab.Formats.JSON;
1 using Implab.Formats.Json;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Globalization;
4 using System.Globalization;
5 using System.Linq;
5 using System.Linq;
6 using System.Text;
7 using System.Threading.Tasks;
8 using System.Xml;
6 using System.Xml;
9
7
10 namespace Implab.Xml {
8 namespace Implab.Xml {
@@ -14,7 +12,7 namespace Implab.Xml {
14 public bool skip;
12 public bool skip;
15 }
13 }
16
14
17 JSONParser m_parser;
15 JsonParser m_parser;
18 JsonXmlReaderOptions m_options;
16 JsonXmlReaderOptions m_options;
19 JsonXmlReaderPosition m_position = JsonXmlReaderPosition.Initial;
17 JsonXmlReaderPosition m_position = JsonXmlReaderPosition.Initial;
20 XmlNameTable m_nameTable;
18 XmlNameTable m_nameTable;
@@ -52,7 +50,6 namespace Implab.Xml {
52
50
53
51
54 XmlNameContext m_context;
52 XmlNameContext m_context;
55 int m_nextPrefix = 1;
56
53
57 readonly string m_xmlnsPrefix;
54 readonly string m_xmlnsPrefix;
58 readonly string m_xmlnsNamespace;
55 readonly string m_xmlnsNamespace;
@@ -60,7 +57,7 namespace Implab.Xml {
60 readonly string m_xsiNamespace;
57 readonly string m_xsiNamespace;
61
58
62
59
63 public JsonXmlReader(JSONParser parser, JsonXmlReaderOptions options) {
60 public JsonXmlReader(JsonParser parser, JsonXmlReaderOptions options) {
64 Safe.ArgumentNotNull(parser, nameof(parser));
61 Safe.ArgumentNotNull(parser, nameof(parser));
65 m_parser = parser;
62 m_parser = parser;
66
63
@@ -480,35 +477,35 namespace Implab.Xml {
480 var jsonName = m_nameTable.Add(m_parser.ElementName);
477 var jsonName = m_nameTable.Add(m_parser.ElementName);
481
478
482 switch (m_parser.ElementType) {
479 switch (m_parser.ElementType) {
483 case JSONElementType.BeginObject:
480 case JsonElementType.BeginObject:
484 if (!EnterJsonObject(jsonName, out elementName))
481 if (!EnterJsonObject(jsonName, out elementName))
485 continue;
482 continue;
486
483
487 m_position = JsonXmlReaderPosition.BeginObject;
484 m_position = JsonXmlReaderPosition.BeginObject;
488 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
485 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
489 break;
486 break;
490 case JSONElementType.EndObject:
487 case JsonElementType.EndObject:
491 if (!LeaveJsonScope(out elementName))
488 if (!LeaveJsonScope(out elementName))
492 continue;
489 continue;
493
490
494 m_position = JsonXmlReaderPosition.EndObject;
491 m_position = JsonXmlReaderPosition.EndObject;
495 EndElementNode(elementName, m_jsonNamespace);
492 EndElementNode(elementName, m_jsonNamespace);
496 break;
493 break;
497 case JSONElementType.BeginArray:
494 case JsonElementType.BeginArray:
498 if (!EnterJsonArray(jsonName, out elementName))
495 if (!EnterJsonArray(jsonName, out elementName))
499 continue;
496 continue;
500
497
501 m_position = JsonXmlReaderPosition.BeginArray;
498 m_position = JsonXmlReaderPosition.BeginArray;
502 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
499 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
503 break;
500 break;
504 case JSONElementType.EndArray:
501 case JsonElementType.EndArray:
505 if (!LeaveJsonScope(out elementName))
502 if (!LeaveJsonScope(out elementName))
506 continue;
503 continue;
507
504
508 m_position = JsonXmlReaderPosition.EndArray;
505 m_position = JsonXmlReaderPosition.EndArray;
509 EndElementNode(elementName, m_jsonNamespace);
506 EndElementNode(elementName, m_jsonNamespace);
510 break;
507 break;
511 case JSONElementType.Value:
508 case JsonElementType.Value:
512 if (!VisitJsonValue(jsonName, out m_jsonValueName))
509 if (!VisitJsonValue(jsonName, out m_jsonValueName))
513 continue;
510 continue;
514
511
Show file before | Show file after | Hide comments
@@ -2,16 +2,16
2 using System;
2 using System;
3 using System.Xml;
3 using System.Xml;
4
4
5 namespace Implab.Formats.JSON {
5 namespace Implab.Xml {
6 /// <summary>
6 /// <summary>
7 /// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
7 /// Набор необязательных параметров для <see cref="JsonXmlReader"/>, позволяющий управлять процессом
8 /// интерпретации <c>JSON</c> документа.
8 /// интерпретации <c>JSON</c> документа.
9 /// </summary>
9 /// </summary>
10 public class JSONXmlReaderOptions : ICloneable {
10 public class JsonXmlReaderOptions : ICloneable {
11 /// <summary>
11 /// <summary>
12 /// Пространство имен в котором будут располагаться читаемые элементы документа
12 /// Пространство имен в котором будут располагаться читаемые элементы документа
13 /// </summary>
13 /// </summary>
14 public string NamespaceURI {
14 public string NamespaceUri {
15 get;
15 get;
16 set;
16 set;
17 }
17 }
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 3
Under Review
sergey
author
Comment #7
Edit
Delete
|

Auto status change to "Under Review"

Approved
sergey
author
Comment #8
Edit
Delete
|

ok, latest stable version should be in default

You need to be logged in to leave comments. Login now