##// END OF EJS Templates
JsonReader optimizations
cin -
r236:302ca905c19e v2
parent child
Show More
@@ -1,147 +1,173
1 using NUnit.Framework;
1 using NUnit.Framework;
2 using System;
2 using System;
3 using Implab.Automaton;
3 using Implab.Automaton;
4 using Implab.Xml;
4 using Implab.Xml;
5 using System.Xml;
5 using System.Xml;
6 using Implab.Formats;
6 using Implab.Formats;
7 using Implab.Formats.Json;
7 using Implab.Formats.Json;
8 using System.IO;
8 using System.IO;
9
9
10 namespace Implab.Format.Test {
10 namespace Implab.Format.Test {
11 [TestFixture]
11 [TestFixture]
12 public class JsonTests {
12 public class JsonTests {
13
13
14 [Test]
14 [Test]
15 public void TestScannerValidTokens() {
15 public void TestScannerValidTokens() {
16 using (var scanner = JsonStringScanner.Create(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
16 using (var scanner = JsonStringScanner.Create(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
17
17
18 Tuple<JsonTokenType, object>[] expexted = {
18 Tuple<JsonTokenType, object>[] expexted = {
19 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "9123"),
19 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "9123"),
20 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
20 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
21 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-123"),
21 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-123"),
22 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
22 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
23 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "0"),
23 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "0"),
24 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
24 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
25 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "0.1"),
25 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "0.1"),
26 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
26 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
27 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-0.2"),
27 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-0.2"),
28 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
28 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
29 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-0.1e3"),
29 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "-0.1e3"),
30 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
30 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
31 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "1.3E-3"),
31 new Tuple<JsonTokenType,object>(JsonTokenType.Number, "1.3E-3"),
32 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
32 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
33 new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n text"),
33 new Tuple<JsonTokenType,object>(JsonTokenType.String, "some \t\n text"),
34 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
34 new Tuple<JsonTokenType,object>(JsonTokenType.ValueSeparator, null),
35 new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal"),
35 new Tuple<JsonTokenType,object>(JsonTokenType.Literal, "literal"),
36 new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, null),
36 new Tuple<JsonTokenType,object>(JsonTokenType.BeginArray, null),
37 new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, null),
37 new Tuple<JsonTokenType,object>(JsonTokenType.EndArray, null),
38 new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, null),
38 new Tuple<JsonTokenType,object>(JsonTokenType.BeginObject, null),
39 new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, null),
39 new Tuple<JsonTokenType,object>(JsonTokenType.EndObject, null),
40 new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, null)
40 new Tuple<JsonTokenType,object>(JsonTokenType.NameSeparator, null)
41 };
41 };
42
42
43 string value;
43 string value;
44 JsonTokenType tokenType;
44 JsonTokenType tokenType;
45 for (var i = 0; i < expexted.Length; i++) {
45 for (var i = 0; i < expexted.Length; i++) {
46
46
47 Assert.IsTrue(scanner.ReadToken(out value, out tokenType));
47 Assert.IsTrue(scanner.ReadToken(out value, out tokenType));
48 Assert.AreEqual(expexted[i].Item1, tokenType);
48 Assert.AreEqual(expexted[i].Item1, tokenType);
49 Assert.AreEqual(expexted[i].Item2, value);
49 Assert.AreEqual(expexted[i].Item2, value);
50 }
50 }
51
51
52 Assert.IsFalse(scanner.ReadToken(out value, out tokenType));
52 Assert.IsFalse(scanner.ReadToken(out value, out tokenType));
53 }
53 }
54 }
54 }
55
55
56 [Test]
56 [Test]
57 public void TestScannerBadTokens() {
57 public void TestScannerBadTokens() {
58 var bad = new[] {
58 var bad = new[] {
59 " 1",
59 " 1",
60 " literal",
60 " literal",
61 " \"",
61 " \"",
62 "\"unclosed string",
62 "\"unclosed string",
63 "1.bad",
63 "1.bad",
64 "001", // should be read as three numbers
64 "001", // should be read as three numbers
65 "--10",
65 "--10",
66 "+10",
66 "+10",
67 "1.0.0",
67 "1.0.0",
68 "1e1.0",
68 "1e1.0",
69 "l1teral0",
69 "l1teral0",
70 ".123",
70 ".123",
71 "-.123"
71 "-.123"
72 };
72 };
73
73
74 foreach (var json in bad) {
74 foreach (var json in bad) {
75 using (var scanner = JsonStringScanner.Create(json)) {
75 using (var scanner = JsonStringScanner.Create(json)) {
76 try {
76 try {
77 string value;
77 string value;
78 JsonTokenType token;
78 JsonTokenType token;
79 scanner.ReadToken(out value, out token);
79 scanner.ReadToken(out value, out token);
80 if (!Object.Equals(value, json)) {
80 if (!Object.Equals(value, json)) {
81 Console.WriteLine("'{0}' is read as {1}", json, value is String ? String.Format("'{0}'", value) : value);
81 Console.WriteLine("'{0}' is read as {1}", json, value is String ? String.Format("'{0}'", value) : value);
82 continue;
82 continue;
83 }
83 }
84 Assert.Fail("Token '{0}' shouldn't pass", json);
84 Assert.Fail("Token '{0}' shouldn't pass", json);
85 } catch (ParserException e) {
85 } catch (ParserException e) {
86 Console.WriteLine(e.Message);
86 Console.WriteLine(e.Message);
87 }
87 }
88 }
88 }
89 }
89 }
90 }
90 }
91
91
92 [Test]
92 [Test]
93 public void JsonXmlReaderSimpleTest() {
93 public void JsonXmlReaderSimpleTest() {
94 var json = "\"some text\"";
94 var json = "\"some text\"";
95 //Console.WriteLine($"JSON: {json}");
95 //Console.WriteLine($"JSON: {json}");
96 //Console.WriteLine("XML");
96 //Console.WriteLine("XML");
97 /*using (var xmlReader = new JsonXmlReader(new JSONParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", RootName = "string", NodesPrefix = "json" })) {
97 /*using (var xmlReader = new JsonXmlReader(new JSONParser(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", RootName = "string", NodesPrefix = "json" })) {
98 Assert.AreEqual(xmlReader.ReadState, System.Xml.ReadState.Initial);
98 Assert.AreEqual(xmlReader.ReadState, System.Xml.ReadState.Initial);
99
99
100 AssertRead(xmlReader, XmlNodeType.XmlDeclaration);
100 AssertRead(xmlReader, XmlNodeType.XmlDeclaration);
101 AssertRead(xmlReader, XmlNodeType.Element);
101 AssertRead(xmlReader, XmlNodeType.Element);
102 AssertRead(xmlReader, XmlNodeType.Text);
102 AssertRead(xmlReader, XmlNodeType.Text);
103 AssertRead(xmlReader, XmlNodeType.EndElement);
103 AssertRead(xmlReader, XmlNodeType.EndElement);
104 Assert.IsFalse(xmlReader.Read());
104 Assert.IsFalse(xmlReader.Read());
105 }*/
105 }*/
106
106
107 //DumpJsonParse("\"text value\"");
107 //DumpJsonParse("\"text value\"");
108 //DumpJsonParse("null");
108 //DumpJsonParse("null");
109 //DumpJsonParse("true");
109 //DumpJsonParse("true");
110 //DumpJsonParse("{}");
110 //DumpJsonParse("{}");
111 //DumpJsonParse("[]");
111 //DumpJsonParse("[]");
112 DumpJsonParse("{\"one\":1, \"two\":2}");
112 DumpJsonParse("{\"one\":1, \"two\":2}");
113 DumpJsonParse("[1,\"\",2,3]");
113 DumpJsonParse("[1,\"\",2,3]");
114 DumpJsonParse("[{\"info\": [7,8,9]}]");
114 DumpJsonParse("[{\"info\": [7,8,9]}]");
115 DumpJsonFlatParse("[1,2,\"\",[3,4],{\"info\": [5,6]},{\"num\": [7,8,null]}, null,[null]]");
115 DumpJsonFlatParse("[1,2,\"\",[3,4],{\"info\": [5,6]},{\"num\": [7,8,null]}, null,[null]]");
116 }
116 }
117
117
118 [Test]
119 public void JsonBenchmark() {
120 var t = Environment.TickCount;
121 using (var reader = new JsonXmlReader(JsonReader.Create("e:\\citylots.json"), new JsonXmlReaderOptions { NamespaceUri = "XmlReaderSimpleTest", RootName = "data" })) {
122 while (reader.Read()) {
123 }
124 }
125 Console.WriteLine($"JsonXmlReader: {Environment.TickCount - t} ms");
126
127 t = Environment.TickCount;
128 using(var reader = JsonReader.Create("e:\\citylots.json")) {
129 while(reader.Read()) {
130 }
131 }
132
133 Console.WriteLine($"JsonReader: {Environment.TickCount - t} ms");
134
135 t = Environment.TickCount;
136 using (var reader = XmlReader.Create("file:///e:\\citylots.xml")) {
137 while (reader.Read()) {
138 }
139 }
140
141 Console.WriteLine($"XmlReader: {Environment.TickCount - t} ms");
142 }
143
118 void AssertRead(XmlReader reader, XmlNodeType expected) {
144 void AssertRead(XmlReader reader, XmlNodeType expected) {
119 Assert.IsTrue(reader.Read());
145 Assert.IsTrue(reader.Read());
120 Console.WriteLine($"{new string(' ', reader.Depth*2)}{reader}");
146 Console.WriteLine($"{new string(' ', reader.Depth * 2)}{reader}");
121 Assert.AreEqual(expected, reader.NodeType);
147 Assert.AreEqual(expected, reader.NodeType);
122 }
148 }
123
149
124 void DumpJsonParse(string json) {
150 void DumpJsonParse(string json) {
125 Console.WriteLine($"JSON: {json}");
151 Console.WriteLine($"JSON: {json}");
126 Console.WriteLine("XML");
152 Console.WriteLine("XML");
127 using (var xmlReader = new JsonXmlReader(JsonReader.ParseString(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "json" })) {
153 using (var xmlReader = new JsonXmlReader(JsonReader.ParseString(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "json" })) {
128 while (xmlReader.Read())
154 while (xmlReader.Read())
129 Console.WriteLine($"{new string(' ', xmlReader.Depth * 2)}{xmlReader}");
155 Console.WriteLine($"{new string(' ', xmlReader.Depth * 2)}{xmlReader}");
130 }
156 }
131 }
157 }
132
158
133 void DumpJsonFlatParse(string json) {
159 void DumpJsonFlatParse(string json) {
134 Console.WriteLine($"JSON: {json}");
160 Console.WriteLine($"JSON: {json}");
135 Console.WriteLine("XML");
161 Console.WriteLine("XML");
136 using (var xmlWriter = XmlWriter.Create(Console.Out, new XmlWriterSettings {
162 using (var xmlWriter = XmlWriter.Create(Console.Out, new XmlWriterSettings {
137 Indent = true,
163 Indent = true,
138 CloseOutput = false,
164 CloseOutput = false,
139 ConformanceLevel = ConformanceLevel.Document
165 ConformanceLevel = ConformanceLevel.Document
140 }))
166 }))
141 using (var xmlReader = new JsonXmlReader(JsonReader.ParseString(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "", FlattenArrays = true })) {
167 using (var xmlReader = new JsonXmlReader(JsonReader.ParseString(json), new JsonXmlReaderOptions { NamespaceUri = "JsonXmlReaderSimpleTest", NodesPrefix = "", FlattenArrays = true })) {
142 xmlWriter.WriteNode(xmlReader, false);
168 xmlWriter.WriteNode(xmlReader, false);
143 }
169 }
144 }
170 }
145 }
171 }
146 }
172 }
147
173
@@ -1,182 +1,92
1 using Implab.Formats.Json;
1 using Implab.Formats.Json;
2 using Implab.Parallels;
2 using Implab.Parallels;
3 using Implab.Xml;
3 using Implab.Xml;
4 using System;
4 using System;
5 using System.Collections.Concurrent;
5 using System.Collections.Concurrent;
6 using System.Collections.Generic;
6 using System.Collections.Generic;
7 using System.IO;
7 using System.IO;
8 using System.Linq;
8 using System.Linq;
9 using System.Text;
9 using System.Text;
10 using System.Threading;
10 using System.Threading;
11 using System.Threading.Tasks;
11 using System.Threading.Tasks;
12 using System.Xml;
12 using System.Xml;
13 using System.Xml.Serialization;
13 using System.Xml.Serialization;
14
14
15 namespace Implab.Playground {
15 namespace Implab.Playground {
16 public class Program {
16 public class Program {
17
17
18 static void EnqueueRange<T>(ConcurrentQueue<T> q, T[] data, int offset, int len) {
18 static void EnqueueRange<T>(ConcurrentQueue<T> q, T[] data, int offset, int len) {
19 for (var i = offset; i < offset + len; i++)
19 for (var i = offset; i < offset + len; i++)
20 q.Enqueue(data[i]);
20 q.Enqueue(data[i]);
21 }
21 }
22
22
23 static bool TryDequeueRange<T>(ConcurrentQueue<T> q,T[] buffer,int offset, int len, out int actual) {
23 static bool TryDequeueRange<T>(ConcurrentQueue<T> q,T[] buffer,int offset, int len, out int actual) {
24 actual = 0;
24 actual = 0;
25 T res;
25 T res;
26 while(q.TryDequeue(out res)) {
26 while(q.TryDequeue(out res)) {
27 buffer[offset + actual] = res;
27 buffer[offset + actual] = res;
28 actual++;
28 actual++;
29 if (actual == len)
29 if (actual == len)
30 break;
30 break;
31 }
31 }
32 return actual != 0;
32 return actual != 0;
33 }
33 }
34
34
35 static void EnqueueRange<T>(SimpleAsyncQueue<T> q, T[] data, int offset, int len) {
35 static void EnqueueRange<T>(SimpleAsyncQueue<T> q, T[] data, int offset, int len) {
36 for (var i = offset; i < offset + len; i++)
36 for (var i = offset; i < offset + len; i++)
37 q.Enqueue(data[i]);
37 q.Enqueue(data[i]);
38 }
38 }
39
39
40 static bool TryDequeueRange<T>(SimpleAsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
40 static bool TryDequeueRange<T>(SimpleAsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
41 actual = 0;
41 actual = 0;
42 T res;
42 T res;
43 while (q.TryDequeue(out res)) {
43 while (q.TryDequeue(out res)) {
44 buffer[offset + actual] = res;
44 buffer[offset + actual] = res;
45 actual++;
45 actual++;
46 if (actual == len)
46 if (actual == len)
47 break;
47 break;
48 }
48 }
49 return actual != 0;
49 return actual != 0;
50 }
50 }
51
51
52 static void EnqueueRange<T>(AsyncQueue<T> q, T[] data, int offset, int len) {
52 static void EnqueueRange<T>(AsyncQueue<T> q, T[] data, int offset, int len) {
53 for (var i = offset; i < offset + len; i++)
53 for (var i = offset; i < offset + len; i++)
54 q.Enqueue(data[i]);
54 q.Enqueue(data[i]);
55 }
55 }
56
56
57 static bool TryDequeueRange<T>(AsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
57 static bool TryDequeueRange<T>(AsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
58 actual = 0;
58 actual = 0;
59 T res;
59 T res;
60 while (q.TryDequeue(out res)) {
60 while (q.TryDequeue(out res)) {
61 buffer[offset + actual] = res;
61 buffer[offset + actual] = res;
62 actual++;
62 actual++;
63 if (actual == len)
63 if (actual == len)
64 break;
64 break;
65 }
65 }
66 return actual != 0;
66 return actual != 0;
67 }
67 }
68
68
69
69
70 /*static void EnqueueRange<T>(AsyncQueue<T> q, T[] data, int offset, int len) {
70 /*static void EnqueueRange<T>(AsyncQueue<T> q, T[] data, int offset, int len) {
71 q.EnqueueRange(data, offset, len);
71 q.EnqueueRange(data, offset, len);
72 }
72 }
73
73
74 static bool TryDequeueRange<T>(AsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
74 static bool TryDequeueRange<T>(AsyncQueue<T> q, T[] buffer, int offset, int len, out int actual) {
75 return q.TryDequeueRange(buffer, offset, len, out actual);
75 return q.TryDequeueRange(buffer, offset, len, out actual);
76 }*/
76 }*/
77
77
78
78
79 static void Main(string[] args) {
79 static void Main(string[] args) {
80
80
81 //var queue = new ConcurrentQueue<int>();
81 var t = Environment.TickCount;
82 var queue = new AsyncQueue<int>();
82 using (var reader = JsonReader.Create("e:\\citylots.json")) {
83 //var queue = new SimpleAsyncQueue<int>();
83 while (reader.Read()) {
84
85 const int wBatch = 32;
86 const long wCount = 1000000;
87 const long total = wBatch * wCount * 3;
88
89 long r1 = 0, r2 = 0, r3 = 0;
90 const int rBatch = 1000;
91 long read = 0;
92
93 var t1 = Environment.TickCount;
94
95 AsyncPool.RunThread(
96 () => {
97 var buffer = new int[wBatch];
98 for (int i = 0; i < wBatch; i++)
99 buffer[i] = 1;
100
101 for (int i = 0; i < wCount; i++)
102 EnqueueRange(queue, buffer, 0, wBatch);
103 Console.WriteLine("done writer #1: {0} ms", Environment.TickCount - t1);
104 },
105 () => {
106 var buffer = new int[wBatch];
107 for (int i = 0; i < wBatch; i++)
108 buffer[i] = 1;
109
110 for (int i = 0; i < wCount; i++)
111 EnqueueRange(queue, buffer, 0, wBatch);
112 Console.WriteLine("done writer #2: {0} ms", Environment.TickCount - t1);
113 },
114 () => {
115 var buffer = new int[wBatch];
116 for (int i = 0; i < wBatch; i++)
117 buffer[i] = 1;
118
119 for (int i = 0; i < wCount; i++)
120 EnqueueRange(queue, buffer, 0, wBatch);
121 Console.WriteLine("done writer #3: {0} ms", Environment.TickCount - t1);
122 },
123 () => {
124 var buffer = new int[rBatch];
125
126 while (read < total) {
127 int actual;
128 if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
129 for (int i = 0; i < actual; i++)
130 r1 += buffer[i];
131 Interlocked.Add(ref read, actual);
132 }
84 }
133 }
85 }
134
86
135 Console.WriteLine("done reader #1: {0} ms", Environment.TickCount - t1);
87 Console.WriteLine($"JsonReader: {Environment.TickCount - t} ms");
136 }/*,
137 () => {
138 var buffer = new int[rBatch];
139
140 while (read < total) {
141 int actual;
142 if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
143 for (int i = 0; i < actual; i++)
144 r2 += buffer[i];
145 Interlocked.Add(ref read, actual);
146 }
147 }
148
149 Console.WriteLine("done reader #2: {0} ms", Environment.TickCount - t1);
150 }*//*,
151 () => {
152 var buffer = new int[rBatch];
153
154 while (read < total) {
155 int actual;
156 if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
157 for (int i = 0; i < actual; i++)
158 r3 += buffer[i];
159 Interlocked.Add(ref read, actual);
160 }
161 }
162
163 Console.WriteLine("done reader #3: {0} ms", Environment.TickCount - t1);
164 }*/
165 )
166 .PromiseAll()
167 .Join();
168
169
170 Console.WriteLine(
171 "done: {0} ms, summ#1: {1}, summ#2: {2}, total: {3}, count: {4}",
172 Environment.TickCount - t1,
173 r1,
174 r2,
175 r1 + r2 + r3,
176 total
177 );
178
88
179 Console.WriteLine("done");
89 Console.WriteLine("done");
180 }
90 }
181 }
91 }
182 }
92 }
@@ -1,9 +1,9
1
1
2 namespace Implab.Automaton {
2 namespace Implab.Automaton {
3 public static class AutomatonConst {
3 public static class AutomatonConst {
4 public const int UNREACHABLE_STATE = -1;
4 public const int UnreachableState = -1;
5
5
6 public const int UNCLASSIFIED_INPUT = 0;
6 public const int UnclassifiedInput = 0;
7 }
7 }
8 }
8 }
9
9
@@ -1,348 +1,348
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5 using System.Diagnostics;
5 using System.Diagnostics;
6 using System.IO;
6 using System.IO;
7 using System.CodeDom.Compiler;
7 using System.CodeDom.Compiler;
8 using System.CodeDom;
8 using System.CodeDom;
9
9
10 namespace Implab.Automaton {
10 namespace Implab.Automaton {
11 public class DFATable : IDFATableBuilder {
11 public class DFATable : IDFATableBuilder {
12 int m_stateCount;
12 int m_stateCount;
13 int m_symbolCount;
13 int m_symbolCount;
14 int m_initialState;
14 int m_initialState;
15
15
16 readonly HashSet<int> m_finalStates = new HashSet<int>();
16 readonly HashSet<int> m_finalStates = new HashSet<int>();
17 readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
17 readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
18
18
19
19
20 #region IDFADefinition implementation
20 #region IDFADefinition implementation
21
21
22 public bool IsFinalState(int s) {
22 public bool IsFinalState(int s) {
23 Safe.ArgumentInRange(s, 0, m_stateCount, "s");
23 Safe.ArgumentInRange(s, 0, m_stateCount, "s");
24
24
25 return m_finalStates.Contains(s);
25 return m_finalStates.Contains(s);
26 }
26 }
27
27
28 public IEnumerable<int> FinalStates {
28 public IEnumerable<int> FinalStates {
29 get {
29 get {
30 return m_finalStates;
30 return m_finalStates;
31 }
31 }
32 }
32 }
33
33
34 public int StateCount {
34 public int StateCount {
35 get { return m_stateCount; }
35 get { return m_stateCount; }
36 }
36 }
37
37
38 public int AlphabetSize {
38 public int AlphabetSize {
39 get { return m_symbolCount; }
39 get { return m_symbolCount; }
40 }
40 }
41
41
42 public int InitialState {
42 public int InitialState {
43 get { return m_initialState; }
43 get { return m_initialState; }
44 }
44 }
45
45
46 #endregion
46 #endregion
47
47
48 public void SetInitialState(int s) {
48 public void SetInitialState(int s) {
49 Safe.ArgumentAssert(s >= 0, "s");
49 Safe.ArgumentAssert(s >= 0, "s");
50 m_stateCount = Math.Max(m_stateCount, s + 1);
50 m_stateCount = Math.Max(m_stateCount, s + 1);
51 m_initialState = s;
51 m_initialState = s;
52 }
52 }
53
53
54 public void MarkFinalState(int state) {
54 public void MarkFinalState(int state) {
55 m_stateCount = Math.Max(m_stateCount, state + 1);
55 m_stateCount = Math.Max(m_stateCount, state + 1);
56 m_finalStates.Add(state);
56 m_finalStates.Add(state);
57 }
57 }
58
58
59 public void Add(AutomatonTransition item) {
59 public void Add(AutomatonTransition item) {
60 Safe.ArgumentAssert(item.s1 >= 0, "item");
60 Safe.ArgumentAssert(item.s1 >= 0, "item");
61 Safe.ArgumentAssert(item.s2 >= 0, "item");
61 Safe.ArgumentAssert(item.s2 >= 0, "item");
62 Safe.ArgumentAssert(item.edge >= 0, "item");
62 Safe.ArgumentAssert(item.edge >= 0, "item");
63
63
64 m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
64 m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
65 m_symbolCount = Math.Max(m_symbolCount, item.edge + 1);
65 m_symbolCount = Math.Max(m_symbolCount, item.edge + 1);
66
66
67 m_transitions.Add(item);
67 m_transitions.Add(item);
68 }
68 }
69
69
70 public void Clear() {
70 public void Clear() {
71 m_stateCount = 0;
71 m_stateCount = 0;
72 m_symbolCount = 0;
72 m_symbolCount = 0;
73 m_finalStates.Clear();
73 m_finalStates.Clear();
74 m_transitions.Clear();
74 m_transitions.Clear();
75 }
75 }
76
76
77 public bool Contains(AutomatonTransition item) {
77 public bool Contains(AutomatonTransition item) {
78 return m_transitions.Contains(item);
78 return m_transitions.Contains(item);
79 }
79 }
80
80
81 public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
81 public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
82 m_transitions.CopyTo(array, arrayIndex);
82 m_transitions.CopyTo(array, arrayIndex);
83 }
83 }
84
84
85 public bool Remove(AutomatonTransition item) {
85 public bool Remove(AutomatonTransition item) {
86 return m_transitions.Remove(item);
86 return m_transitions.Remove(item);
87 }
87 }
88
88
89 public int Count {
89 public int Count {
90 get {
90 get {
91 return m_transitions.Count;
91 return m_transitions.Count;
92 }
92 }
93 }
93 }
94
94
95 public bool IsReadOnly {
95 public bool IsReadOnly {
96 get {
96 get {
97 return false;
97 return false;
98 }
98 }
99 }
99 }
100
100
101 public IEnumerator<AutomatonTransition> GetEnumerator() {
101 public IEnumerator<AutomatonTransition> GetEnumerator() {
102 return m_transitions.GetEnumerator();
102 return m_transitions.GetEnumerator();
103 }
103 }
104
104
105 System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
105 System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
106 return GetEnumerator();
106 return GetEnumerator();
107 }
107 }
108
108
109 public void AddSymbol(int symbol) {
109 public void AddSymbol(int symbol) {
110 Safe.ArgumentAssert(symbol >= 0, "symbol");
110 Safe.ArgumentAssert(symbol >= 0, "symbol");
111 m_symbolCount = Math.Max(symbol + 1, m_symbolCount);
111 m_symbolCount = Math.Max(symbol + 1, m_symbolCount);
112 }
112 }
113
113
114 public int[,] CreateTransitionTable() {
114 public int[,] CreateTransitionTable() {
115 var table = new int[StateCount,AlphabetSize];
115 var table = new int[StateCount,AlphabetSize];
116
116
117 for (int i = 0; i < StateCount; i++)
117 for (int i = 0; i < StateCount; i++)
118 for (int j = 0; j < AlphabetSize; j++)
118 for (int j = 0; j < AlphabetSize; j++)
119 table[i, j] = AutomatonConst.UNREACHABLE_STATE;
119 table[i, j] = AutomatonConst.UnreachableState;
120
120
121 foreach (var t in this)
121 foreach (var t in this)
122 table[t.s1,t.edge] = (byte)t.s2;
122 table[t.s1,t.edge] = (byte)t.s2;
123
123
124 return table;
124 return table;
125 }
125 }
126
126
127 public bool[] CreateFinalStateTable() {
127 public bool[] CreateFinalStateTable() {
128 var table = new bool[StateCount];
128 var table = new bool[StateCount];
129
129
130 foreach (var s in FinalStates)
130 foreach (var s in FinalStates)
131 table[s] = true;
131 table[s] = true;
132
132
133 return table;
133 return table;
134 }
134 }
135
135
136 /// <summary>Π€ΠΎΡ€ΠΌΠΈΡ€ΡƒΠ΅Ρ‚ мноТСства ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… состояний ΠΏΠ΅Ρ€Π΅Π΄ Π½Π°Ρ‡Π°Π»ΠΎΠΌ Ρ€Π°Π±ΠΎΡ‚Ρ‹ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌΠ° ΠΌΠΈΠ½ΠΈΠΌΠΈΠ·Π°Ρ†ΠΈΠΈ.</summary>
136 /// <summary>Π€ΠΎΡ€ΠΌΠΈΡ€ΡƒΠ΅Ρ‚ мноТСства ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… состояний ΠΏΠ΅Ρ€Π΅Π΄ Π½Π°Ρ‡Π°Π»ΠΎΠΌ Ρ€Π°Π±ΠΎΡ‚Ρ‹ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌΠ° ΠΌΠΈΠ½ΠΈΠΌΠΈΠ·Π°Ρ†ΠΈΠΈ.</summary>
137 /// <remarks>
137 /// <remarks>
138 /// Π’ процСссС построСния минимального Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π° трСбуСтся Ρ€Π°Π·Π΄Π΅Π»ΠΈΡ‚ΡŒ мноТСство состояний,
138 /// Π’ процСссС построСния минимального Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π° трСбуСтся Ρ€Π°Π·Π΄Π΅Π»ΠΈΡ‚ΡŒ мноТСство состояний,
139 /// Π½Π° Π΄Π²Π° подмноТСства - ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Π΅ состояния ΠΈ всС ΠΎΡΡ‚Π°Π»ΡŒΠ½Ρ‹Π΅, послС Ρ‡Π΅Π³ΠΎ эти подмноТСства
139 /// Π½Π° Π΄Π²Π° подмноТСства - ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Π΅ состояния ΠΈ всС ΠΎΡΡ‚Π°Π»ΡŒΠ½Ρ‹Π΅, послС Ρ‡Π΅Π³ΠΎ эти подмноТСства
140 /// Π±ΡƒΠ΄ΡƒΡ‚ Ρ€Π΅Π·Π΄Π΅Π»Π΅Π½Ρ‹ Π½Π° Π±ΠΎΠ»Π΅Π΅ ΠΌΠ΅Π»ΠΊΠΈΠ΅. Иногда трСбуСтся Π³Π°Ρ€Π°Π½Ρ‚ΠΈΡ€ΠΎΠ²Π°Ρ‚ΡŒ различия ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… сосотяний,
140 /// Π±ΡƒΠ΄ΡƒΡ‚ Ρ€Π΅Π·Π΄Π΅Π»Π΅Π½Ρ‹ Π½Π° Π±ΠΎΠ»Π΅Π΅ ΠΌΠ΅Π»ΠΊΠΈΠ΅. Иногда трСбуСтся Π³Π°Ρ€Π°Π½Ρ‚ΠΈΡ€ΠΎΠ²Π°Ρ‚ΡŒ различия ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… сосотяний,
141 /// для этого Π½Π΅ΠΎΠ±Ρ…ΠΎΠ΄ΠΈΠΌΠΎ ΠΏΠ΅Ρ€Π΅ΠΎΠΏΡ€Π΅Π΄Π΅Π»ΠΈΡ‚ΡŒ Π΄Π°Π½Π½Ρ†ΡŽ Ρ„ΡƒΠΊΠ½Ρ†ΠΈΡŽ, для получСния мноТСств ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… состояний.
141 /// для этого Π½Π΅ΠΎΠ±Ρ…ΠΎΠ΄ΠΈΠΌΠΎ ΠΏΠ΅Ρ€Π΅ΠΎΠΏΡ€Π΅Π΄Π΅Π»ΠΈΡ‚ΡŒ Π΄Π°Π½Π½Ρ†ΡŽ Ρ„ΡƒΠΊΠ½Ρ†ΠΈΡŽ, для получСния мноТСств ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Ρ… состояний.
142 /// </remarks>
142 /// </remarks>
143 /// <returns>The final states.</returns>
143 /// <returns>The final states.</returns>
144 protected virtual IEnumerable<HashSet<int>> SplitFinalStates(IEnumerable<int> states) {
144 protected virtual IEnumerable<HashSet<int>> SplitFinalStates(IEnumerable<int> states) {
145 return new [] { new HashSet<int>(states) };
145 return new [] { new HashSet<int>(states) };
146 }
146 }
147
147
148 protected void Optimize(
148 protected void Optimize(
149 IDFATableBuilder optimalDFA,
149 IDFATableBuilder optimalDFA,
150 IDictionary<int,int> alphabetMap,
150 IDictionary<int,int> alphabetMap,
151 IDictionary<int,int> stateMap
151 IDictionary<int,int> stateMap
152 ) {
152 ) {
153 Safe.ArgumentNotNull(optimalDFA, "dfa");
153 Safe.ArgumentNotNull(optimalDFA, "dfa");
154 Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
154 Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
155 Safe.ArgumentNotNull(stateMap, "stateMap");
155 Safe.ArgumentNotNull(stateMap, "stateMap");
156
156
157
157
158 var setComparer = new CustomEqualityComparer<HashSet<int>>(
158 var setComparer = new CustomEqualityComparer<HashSet<int>>(
159 (x, y) => x.SetEquals(y),
159 (x, y) => x.SetEquals(y),
160 s => s.Sum(x => x.GetHashCode())
160 s => s.Sum(x => x.GetHashCode())
161 );
161 );
162
162
163 var optimalStates = new HashSet<HashSet<int>>(setComparer);
163 var optimalStates = new HashSet<HashSet<int>>(setComparer);
164 var queue = new HashSet<HashSet<int>>(setComparer);
164 var queue = new HashSet<HashSet<int>>(setComparer);
165
165
166 optimalStates.Add(new HashSet<int>(FinalStates));
166 optimalStates.Add(new HashSet<int>(FinalStates));
167
167
168 var state = new HashSet<int>(
168 var state = new HashSet<int>(
169 Enumerable
169 Enumerable
170 .Range(0, m_stateCount)
170 .Range(0, m_stateCount)
171 .Where(i => !m_finalStates.Contains(i))
171 .Where(i => !m_finalStates.Contains(i))
172 );
172 );
173
173
174 optimalStates.Add(state);
174 optimalStates.Add(state);
175 queue.Add(state);
175 queue.Add(state);
176
176
177 var rmap = m_transitions
177 var rmap = m_transitions
178 .GroupBy(t => t.s2)
178 .GroupBy(t => t.s2)
179 .ToDictionary(
179 .ToDictionary(
180 g => g.Key, // s2
180 g => g.Key, // s2
181 g => g.ToLookup(t => t.edge, t => t.s1)//.ToDictionary(p => p.Key)
181 g => g.ToLookup(t => t.edge, t => t.s1)//.ToDictionary(p => p.Key)
182 );
182 );
183
183
184 while (queue.Count > 0) {
184 while (queue.Count > 0) {
185 var stateA = queue.First();
185 var stateA = queue.First();
186 queue.Remove(stateA);
186 queue.Remove(stateA);
187
187
188 for (int c = 0; c < m_symbolCount; c++) {
188 for (int c = 0; c < m_symbolCount; c++) {
189 var stateX = new HashSet<int>();
189 var stateX = new HashSet<int>();
190 foreach(var a in stateA.Where(rmap.ContainsKey))
190 foreach(var a in stateA.Where(rmap.ContainsKey))
191 stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
191 stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
192
192
193 var tmp = optimalStates.ToArray();
193 var tmp = optimalStates.ToArray();
194 foreach (var stateY in tmp) {
194 foreach (var stateY in tmp) {
195 var stateR1 = new HashSet<int>(stateY);
195 var stateR1 = new HashSet<int>(stateY);
196 var stateR2 = new HashSet<int>(stateY);
196 var stateR2 = new HashSet<int>(stateY);
197
197
198 stateR1.IntersectWith(stateX);
198 stateR1.IntersectWith(stateX);
199 stateR2.ExceptWith(stateX);
199 stateR2.ExceptWith(stateX);
200
200
201 if (stateR1.Count > 0 && stateR2.Count > 0) {
201 if (stateR1.Count > 0 && stateR2.Count > 0) {
202
202
203
203
204 optimalStates.Remove(stateY);
204 optimalStates.Remove(stateY);
205 optimalStates.Add(stateR1);
205 optimalStates.Add(stateR1);
206 optimalStates.Add(stateR2);
206 optimalStates.Add(stateR2);
207
207
208 if (queue.Contains(stateY)) {
208 if (queue.Contains(stateY)) {
209 queue.Remove(stateY);
209 queue.Remove(stateY);
210 queue.Add(stateR1);
210 queue.Add(stateR1);
211 queue.Add(stateR2);
211 queue.Add(stateR2);
212 } else {
212 } else {
213 queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
213 queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
214 }
214 }
215 }
215 }
216 }
216 }
217 }
217 }
218 }
218 }
219
219
220 // Π΄ΠΎΠΏΠΎΠ»Π½ΠΈΡ‚Π΅Π»ΡŒΠ½ΠΎ Ρ€Π°Π·Π±ΠΈΠ²Π°Π΅ΠΌ ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Π΅ состояния
220 // Π΄ΠΎΠΏΠΎΠ»Π½ΠΈΡ‚Π΅Π»ΡŒΠ½ΠΎ Ρ€Π°Π·Π±ΠΈΠ²Π°Π΅ΠΌ ΠΊΠΎΠ½Π΅Ρ‡Π½Ρ‹Π΅ состояния
221 foreach (var final in optimalStates.Where(s => s.Overlaps(m_finalStates)).ToArray()) {
221 foreach (var final in optimalStates.Where(s => s.Overlaps(m_finalStates)).ToArray()) {
222 optimalStates.Remove(final);
222 optimalStates.Remove(final);
223 foreach (var split in SplitFinalStates(final))
223 foreach (var split in SplitFinalStates(final))
224 optimalStates.Add(split);
224 optimalStates.Add(split);
225 }
225 }
226
226
227
227
228 // ΠΊΠ°Ρ€Ρ‚Π° получСния ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½ΠΎΠ³ΠΎ состояния ΠΏΠΎ ΡΠΎΠΎΡ‚Π²Π΅ΡΡ‚Π²ΡƒΡŽΡ‰Π΅ΠΌΡƒ Π΅ΠΌΡƒ простому ΡΠΎΡΡ‚ΠΎΡΠ½ΠΈΡŽ
228 // ΠΊΠ°Ρ€Ρ‚Π° получСния ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½ΠΎΠ³ΠΎ состояния ΠΏΠΎ ΡΠΎΠΎΡ‚Π²Π΅ΡΡ‚Π²ΡƒΡŽΡ‰Π΅ΠΌΡƒ Π΅ΠΌΡƒ простому ΡΠΎΡΡ‚ΠΎΡΠ½ΠΈΡŽ
229 var nextState = 0;
229 var nextState = 0;
230 foreach (var item in optimalStates) {
230 foreach (var item in optimalStates) {
231 var id = nextState++;
231 var id = nextState++;
232 foreach (var s in item)
232 foreach (var s in item)
233 stateMap[s] = id;
233 stateMap[s] = id;
234 }
234 }
235
235
236 // ΠΏΠΎΠ»ΡƒΡ‡Π°Π΅ΠΌ ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ Π°Π»Ρ„Π°Π²ΠΈΡ‚
236 // ΠΏΠΎΠ»ΡƒΡ‡Π°Π΅ΠΌ ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ Π°Π»Ρ„Π°Π²ΠΈΡ‚
237 // Π²Ρ…ΠΎΠ΄Π½Ρ‹Π΅ символы Π½Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹, Ссли Move(s,a1) == Move(s,a2), для любого s
237 // Π²Ρ…ΠΎΠ΄Π½Ρ‹Π΅ символы Π½Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹, Ссли Move(s,a1) == Move(s,a2), для любого s
238 // для этого ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌ кластСризации, сначала
238 // для этого ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌ кластСризации, сначала
239 // считаСм, Ρ‡Ρ‚ΠΎ всС символы Π½Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹
239 // считаСм, Ρ‡Ρ‚ΠΎ всС символы Π½Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹
240
240
241 var minClasses = new HashSet<HashSet<int>>(setComparer);
241 var minClasses = new HashSet<HashSet<int>>(setComparer);
242 var alphaQueue = new Queue<HashSet<int>>();
242 var alphaQueue = new Queue<HashSet<int>>();
243 alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
243 alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
244
244
245 // для всСх состояний, Π±ΡƒΠ΄Π΅ΠΌ ΠΏΡ€ΠΎΠ²Π΅Ρ€ΡΡ‚ΡŒ ΠΊΠ°ΠΆΠ΄Ρ‹ΠΉ класс Π½Π° Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΠΎΡΡ‚ΡŒ,
245 // для всСх состояний, Π±ΡƒΠ΄Π΅ΠΌ ΠΏΡ€ΠΎΠ²Π΅Ρ€ΡΡ‚ΡŒ ΠΊΠ°ΠΆΠ΄Ρ‹ΠΉ класс Π½Π° Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΠΎΡΡ‚ΡŒ,
246 // Ρ‚.Π΅. символы Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹, Ссли ΠΎΠ½ΠΈ приводят ΠΊ Ρ€Π°Π·Π½Ρ‹ΠΌ состояниям
246 // Ρ‚.Π΅. символы Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹, Ссли ΠΎΠ½ΠΈ приводят ΠΊ Ρ€Π°Π·Π½Ρ‹ΠΌ состояниям
247 for (int s = 0 ; s < optimalStates.Count; s++) {
247 for (int s = 0 ; s < optimalStates.Count; s++) {
248 var newQueue = new Queue<HashSet<int>>();
248 var newQueue = new Queue<HashSet<int>>();
249
249
250 foreach (var A in alphaQueue) {
250 foreach (var A in alphaQueue) {
251 // классы ΠΈΠ· ΠΎΠ΄Π½ΠΎΠ³ΠΎ символа Π΄Π΅Π»ΠΈΡ‚ΡŒ бСсполСзно, ΠΏΠ΅Ρ€Π΅Π²ΠΎΠ΄ΠΈΠΌ ΠΈΡ… сразу Π²
251 // классы ΠΈΠ· ΠΎΠ΄Π½ΠΎΠ³ΠΎ символа Π΄Π΅Π»ΠΈΡ‚ΡŒ бСсполСзно, ΠΏΠ΅Ρ€Π΅Π²ΠΎΠ΄ΠΈΠΌ ΠΈΡ… сразу Π²
252 // Ρ€Π΅Π·ΡƒΠ»ΡŒΡ‚ΠΈΡ€ΡƒΡŽΡ‰ΠΈΠΉ Π°Π»Ρ„Π°Π²ΠΈΡ‚
252 // Ρ€Π΅Π·ΡƒΠ»ΡŒΡ‚ΠΈΡ€ΡƒΡŽΡ‰ΠΈΠΉ Π°Π»Ρ„Π°Π²ΠΈΡ‚
253 if (A.Count == 1) {
253 if (A.Count == 1) {
254 minClasses.Add(A);
254 minClasses.Add(A);
255 continue;
255 continue;
256 }
256 }
257
257
258 // Ρ€Π°Π·Π»ΠΈΡ‡Π°Π΅ΠΌ классы символов, ΠΊΠΎΡ‚ΠΎΡ€Ρ‹Π΅ пСрСводят Π² Ρ€Π°Π·Π»ΠΈΡ‡Π½Ρ‹Π΅ ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹Π΅ состояния
258 // Ρ€Π°Π·Π»ΠΈΡ‡Π°Π΅ΠΌ классы символов, ΠΊΠΎΡ‚ΠΎΡ€Ρ‹Π΅ пСрСводят Π² Ρ€Π°Π·Π»ΠΈΡ‡Π½Ρ‹Π΅ ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹Π΅ состояния
259 // optimalState -> alphaClass
259 // optimalState -> alphaClass
260 var classes = new Dictionary<int, HashSet<int>>();
260 var classes = new Dictionary<int, HashSet<int>>();
261
261
262 foreach (var term in A) {
262 foreach (var term in A) {
263 // ΠΈΡ‰Π΅ΠΌ всС ΠΏΠ΅Ρ€Π΅Ρ…ΠΎΠ΄Ρ‹ класса ΠΏΠΎ символу term
263 // ΠΈΡ‰Π΅ΠΌ всС ΠΏΠ΅Ρ€Π΅Ρ…ΠΎΠ΄Ρ‹ класса ΠΏΠΎ символу term
264 var s2 = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).DefaultIfEmpty(-1).First();
264 var s2 = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).DefaultIfEmpty(-1).First();
265
265
266 HashSet<int> a2;
266 HashSet<int> a2;
267 if (!classes.TryGetValue(s2, out a2)) {
267 if (!classes.TryGetValue(s2, out a2)) {
268 a2 = new HashSet<int>();
268 a2 = new HashSet<int>();
269 newQueue.Enqueue(a2);
269 newQueue.Enqueue(a2);
270 classes[s2] = a2;
270 classes[s2] = a2;
271 }
271 }
272 a2.Add(term);
272 a2.Add(term);
273 }
273 }
274 }
274 }
275
275
276 if (newQueue.Count == 0)
276 if (newQueue.Count == 0)
277 break;
277 break;
278 alphaQueue = newQueue;
278 alphaQueue = newQueue;
279 }
279 }
280
280
281 // послС окончания Ρ€Π°Π±ΠΎΡ‚Ρ‹ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌΠ° Π² ΠΎΡ‡Π΅Ρ€Π΅Π΄ΠΈ останутся ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹Π΅ классы
281 // послС окончания Ρ€Π°Π±ΠΎΡ‚Ρ‹ Π°Π»Π³ΠΎΡ€ΠΈΡ‚ΠΌΠ° Π² ΠΎΡ‡Π΅Ρ€Π΅Π΄ΠΈ останутся ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹Π΅ Ρ€Π°Π·Π»ΠΈΡ‡ΠΈΠΌΡ‹Π΅ классы
282 // Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… символов
282 // Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… символов
283 foreach (var A in alphaQueue)
283 foreach (var A in alphaQueue)
284 minClasses.Add(A);
284 minClasses.Add(A);
285
285
286 // построСниС отобраТСния Π°Π»Ρ„Π°Π²ΠΈΡ‚ΠΎΠ² Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… символов.
286 // построСниС отобраТСния Π°Π»Ρ„Π°Π²ΠΈΡ‚ΠΎΠ² Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… символов.
287 // ΠΏΠΎΡΠΊΠΎΠ»ΡŒΠΊΡƒ символ DFAConst.UNCLASSIFIED_INPUT ΠΌΠΎΠΆΠ΅Ρ‚ ΠΈΠΌΠ΅Ρ‚ΡŒ
287 // ΠΏΠΎΡΠΊΠΎΠ»ΡŒΠΊΡƒ символ DFAConst.UNCLASSIFIED_INPUT ΠΌΠΎΠΆΠ΅Ρ‚ ΠΈΠΌΠ΅Ρ‚ΡŒ
288 // ΡΠΏΠ΅Ρ†ΠΈΠ°Π»ΡŒΠ½ΠΎΠ΅ Π·Π½Π°Ρ‡Π΅Π½ΠΈΠ΅, Ρ‚ΠΎΠ³Π΄Π° сохраним ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ класс,
288 // ΡΠΏΠ΅Ρ†ΠΈΠ°Π»ΡŒΠ½ΠΎΠ΅ Π·Π½Π°Ρ‡Π΅Π½ΠΈΠ΅, Ρ‚ΠΎΠ³Π΄Π° сохраним ΠΌΠΈΠ½ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ класс,
289 // содСрТащий этот символ Π½Π° Ρ‚ΠΎΠΌΠΆΠ΅ мСстС.
289 // содСрТащий этот символ Π½Π° Ρ‚ΠΎΠΌΠΆΠ΅ мСстС.
290
290
291 var nextCls = 0;
291 var nextCls = 0;
292 foreach (var item in minClasses) {
292 foreach (var item in minClasses) {
293 if (nextCls == AutomatonConst.UNCLASSIFIED_INPUT)
293 if (nextCls == AutomatonConst.UnclassifiedInput)
294 nextCls++;
294 nextCls++;
295
295
296 // сохраняСм DFAConst.UNCLASSIFIED_INPUT
296 // сохраняСм DFAConst.UNCLASSIFIED_INPUT
297 var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++;
297 var cls = item.Contains(AutomatonConst.UnclassifiedInput) ? AutomatonConst.UnclassifiedInput : nextCls++;
298 optimalDFA.AddSymbol(cls);
298 optimalDFA.AddSymbol(cls);
299
299
300 foreach (var a in item)
300 foreach (var a in item)
301 alphabetMap[a] = cls;
301 alphabetMap[a] = cls;
302 }
302 }
303
303
304 // построСниС Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π°
304 // построСниС Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π°
305 optimalDFA.SetInitialState(stateMap[m_initialState]);
305 optimalDFA.SetInitialState(stateMap[m_initialState]);
306
306
307 foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
307 foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
308 optimalDFA.MarkFinalState(sf);
308 optimalDFA.MarkFinalState(sf);
309
309
310 foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
310 foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
311 optimalDFA.Add(t);
311 optimalDFA.Add(t);
312 }
312 }
313
313
314 protected string PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
314 protected string PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
315 Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
315 Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
316 Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
316 Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
317
317
318 var data = new List<string>();
318 var data = new List<string>();
319
319
320 data.Add("digraph dfa {");
320 data.Add("digraph dfa {");
321
321
322 foreach (var final in m_finalStates)
322 foreach (var final in m_finalStates)
323 data.Add(String.Format("{0} [shape=box];",String.Join("", stateAlphabet.GetSymbols(final))));
323 data.Add(String.Format("{0} [shape=box];",String.Join("", stateAlphabet.GetSymbols(final))));
324
324
325 foreach (var t in m_transitions)
325 foreach (var t in m_transitions)
326 data.Add(String.Format(
326 data.Add(String.Format(
327 "{0} -> {2} [label={1}];",
327 "{0} -> {2} [label={1}];",
328 String.Join("", stateAlphabet.GetSymbols(t.s1)),
328 String.Join("", stateAlphabet.GetSymbols(t.s1)),
329 ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UNCLASSIFIED_INPUT ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
329 ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UnclassifiedInput ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
330 String.Join("", stateAlphabet.GetSymbols(t.s2))
330 String.Join("", stateAlphabet.GetSymbols(t.s2))
331 ));
331 ));
332 data.Add("}");
332 data.Add("}");
333 return String.Join("\n", data);
333 return String.Join("\n", data);
334 }
334 }
335
335
336 static string ToLiteral(string input)
336 static string ToLiteral(string input)
337 {
337 {
338 using (var writer = new StringWriter())
338 using (var writer = new StringWriter())
339 {
339 {
340 using (var provider = CodeDomProvider.CreateProvider("CSharp"))
340 using (var provider = CodeDomProvider.CreateProvider("CSharp"))
341 {
341 {
342 provider.GenerateCodeFromExpression(new CodePrimitiveExpression(input), writer, null);
342 provider.GenerateCodeFromExpression(new CodePrimitiveExpression(input), writer, null);
343 return writer.ToString();
343 return writer.ToString();
344 }
344 }
345 }
345 }
346 }
346 }
347 }
347 }
348 }
348 }
@@ -1,84 +1,84
1 using System;
1 using System;
2 using System.Collections.Generic;
2 using System.Collections.Generic;
3 using System.Linq;
3 using System.Linq;
4
4
5 namespace Implab.Automaton {
5 namespace Implab.Automaton {
6 public class MapAlphabet<T> : IAlphabetBuilder<T> {
6 public class MapAlphabet<T> : IAlphabetBuilder<T> {
7 readonly Dictionary<T,int> m_map;
7 readonly Dictionary<T,int> m_map;
8 int m_nextCls;
8 int m_nextCls;
9 readonly bool m_supportUnclassified;
9 readonly bool m_supportUnclassified;
10
10
11 public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
11 public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
12 m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
12 m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
13 m_supportUnclassified = supportUnclassified;
13 m_supportUnclassified = supportUnclassified;
14 m_nextCls = supportUnclassified ? 1 : 0;
14 m_nextCls = supportUnclassified ? 1 : 0;
15 }
15 }
16
16
17 #region IAlphabetBuilder implementation
17 #region IAlphabetBuilder implementation
18
18
19 public int DefineSymbol(T symbol) {
19 public int DefineSymbol(T symbol) {
20 int cls;
20 int cls;
21 return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
21 return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
22 }
22 }
23
23
24 public int DefineSymbol(T symbol, int cls) {
24 public int DefineSymbol(T symbol, int cls) {
25 Safe.ArgumentAssert(cls >= 0, "cls");
25 Safe.ArgumentAssert(cls >= 0, "cls");
26
26
27 m_nextCls = Math.Max(cls + 1, m_nextCls);
27 m_nextCls = Math.Max(cls + 1, m_nextCls);
28 m_map.Add(symbol, cls);
28 m_map.Add(symbol, cls);
29 return cls;
29 return cls;
30 }
30 }
31
31
32 public int DefineClass(IEnumerable<T> symbols) {
32 public int DefineClass(IEnumerable<T> symbols) {
33 return DefineClass(symbols, m_nextCls);
33 return DefineClass(symbols, m_nextCls);
34 }
34 }
35
35
36 public int DefineClass(IEnumerable<T> symbols, int cls) {
36 public int DefineClass(IEnumerable<T> symbols, int cls) {
37 Safe.ArgumentAssert(cls >= 0, "cls");
37 Safe.ArgumentAssert(cls >= 0, "cls");
38 Safe.ArgumentNotNull(symbols, "symbols");
38 Safe.ArgumentNotNull(symbols, "symbols");
39
39
40 m_nextCls = Math.Max(cls + 1, m_nextCls);
40 m_nextCls = Math.Max(cls + 1, m_nextCls);
41
41
42 foreach (var symbol in symbols)
42 foreach (var symbol in symbols)
43 m_map[symbol] = cls;
43 m_map[symbol] = cls;
44 return cls;
44 return cls;
45 }
45 }
46
46
47 #endregion
47 #endregion
48
48
49 #region IAlphabet implementation
49 #region IAlphabet implementation
50
50
51 public int Translate(T symbol) {
51 public int Translate(T symbol) {
52 int cls;
52 int cls;
53 if (m_map.TryGetValue(symbol, out cls))
53 if (m_map.TryGetValue(symbol, out cls))
54 return cls;
54 return cls;
55 if (!m_supportUnclassified)
55 if (!m_supportUnclassified)
56 throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
56 throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
57 return AutomatonConst.UNCLASSIFIED_INPUT;
57 return AutomatonConst.UnclassifiedInput;
58 }
58 }
59
59
60 public int Count {
60 public int Count {
61 get {
61 get {
62 return m_nextCls;
62 return m_nextCls;
63 }
63 }
64 }
64 }
65
65
66 public bool Contains(T symbol) {
66 public bool Contains(T symbol) {
67 return m_supportUnclassified || m_map.ContainsKey(symbol);
67 return m_supportUnclassified || m_map.ContainsKey(symbol);
68 }
68 }
69
69
70
70
71 public IEnumerable<T> GetSymbols(int cls) {
71 public IEnumerable<T> GetSymbols(int cls) {
72 Safe.ArgumentAssert(!m_supportUnclassified || cls > 0, "cls");
72 Safe.ArgumentAssert(!m_supportUnclassified || cls > 0, "cls");
73 return m_map.Where(p => p.Value == cls).Select(p => p.Key);
73 return m_map.Where(p => p.Value == cls).Select(p => p.Key);
74 }
74 }
75 #endregion
75 #endregion
76
76
77 public IEnumerable<KeyValuePair<T,int>> Mappings {
77 public IEnumerable<KeyValuePair<T,int>> Mappings {
78 get {
78 get {
79 return m_map;
79 return m_map;
80 }
80 }
81 }
81 }
82 }
82 }
83 }
83 }
84
84
@@ -1,212 +1,212
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Diagnostics;
4 using System.Diagnostics;
5 using System.Linq;
5 using System.Linq;
6
6
7 namespace Implab.Automaton.RegularExpressions {
7 namespace Implab.Automaton.RegularExpressions {
8 /// <summary>
8 /// <summary>
9 /// Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅Ρ‚ΡΡ для построСния Π”ΠšΠ ΠΏΠΎ рСгулярному Π²Ρ‹Ρ€Π°ΠΆΠ΅Π½ΠΈΡŽ, сначала ΠΎΠ±Ρ…ΠΎΠ΄ΠΈΡ‚
9 /// Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅Ρ‚ΡΡ для построСния Π”ΠšΠ ΠΏΠΎ рСгулярному Π²Ρ‹Ρ€Π°ΠΆΠ΅Π½ΠΈΡŽ, сначала ΠΎΠ±Ρ…ΠΎΠ΄ΠΈΡ‚
10 /// рСгулярноС Π²Ρ‹Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ ΠΈ вычисляСт followpos, Π·Π°Ρ‚Π΅ΠΌ ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅Ρ‚ΡΡ ΠΌΠ΅Ρ‚ΠΎΠ΄
10 /// рСгулярноС Π²Ρ‹Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ ΠΈ вычисляСт followpos, Π·Π°Ρ‚Π΅ΠΌ ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅Ρ‚ΡΡ ΠΌΠ΅Ρ‚ΠΎΠ΄
11 /// <see cref="BuildDFA(IDFADefinition)"/> для построСния Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π°.
11 /// <see cref="BuildDFA(IDFADefinition)"/> для построСния Π°Π²Ρ‚ΠΎΠΌΠ°Ρ‚Π°.
12 /// </summary>
12 /// </summary>
13 public class RegularExpressionVisitor : IVisitor {
13 public class RegularExpressionVisitor : IVisitor {
14 int m_idx;
14 int m_idx;
15 Token m_root;
15 Token m_root;
16 HashSet<int> m_firstpos;
16 HashSet<int> m_firstpos;
17 HashSet<int> m_lastpos;
17 HashSet<int> m_lastpos;
18
18
19 readonly Dictionary<int, HashSet<int>> m_followpos = new Dictionary<int, HashSet<int>>();
19 readonly Dictionary<int, HashSet<int>> m_followpos = new Dictionary<int, HashSet<int>>();
20 readonly Dictionary<int, int> m_indexes = new Dictionary<int, int>();
20 readonly Dictionary<int, int> m_indexes = new Dictionary<int, int>();
21 readonly HashSet<int> m_ends = new HashSet<int>();
21 readonly HashSet<int> m_ends = new HashSet<int>();
22
22
23 readonly IDFATableBuilder m_builder;
23 readonly IDFATableBuilder m_builder;
24 readonly IAlphabetBuilder<HashSet<int>> m_states = new MapAlphabet<HashSet<int>>(
24 readonly IAlphabetBuilder<HashSet<int>> m_states = new MapAlphabet<HashSet<int>>(
25 false,
25 false,
26 new CustomEqualityComparer<HashSet<int>>(
26 new CustomEqualityComparer<HashSet<int>>(
27 (x, y) => x.SetEquals(y),
27 (x, y) => x.SetEquals(y),
28 x => x.Sum(n => n.GetHashCode())
28 x => x.Sum(n => n.GetHashCode())
29 )
29 )
30 );
30 );
31
31
32 public RegularExpressionVisitor(IDFATableBuilder builder) {
32 public RegularExpressionVisitor(IDFATableBuilder builder) {
33 Safe.ArgumentNotNull(builder, "builder");
33 Safe.ArgumentNotNull(builder, "builder");
34
34
35 m_builder = builder;
35 m_builder = builder;
36 }
36 }
37
37
38 HashSet<int> Followpos(int pos) {
38 HashSet<int> Followpos(int pos) {
39 HashSet<int> set;
39 HashSet<int> set;
40 return m_followpos.TryGetValue(pos, out set) ? set : m_followpos[pos] = new HashSet<int>();
40 return m_followpos.TryGetValue(pos, out set) ? set : m_followpos[pos] = new HashSet<int>();
41 }
41 }
42
42
43 bool Nullable(object n) {
43 bool Nullable(object n) {
44 if (n is EmptyToken || n is StarToken)
44 if (n is EmptyToken || n is StarToken)
45 return true;
45 return true;
46 var altToken = n as AltToken;
46 var altToken = n as AltToken;
47 if (altToken != null)
47 if (altToken != null)
48 return Nullable(altToken.Left) || Nullable(altToken.Right);
48 return Nullable(altToken.Left) || Nullable(altToken.Right);
49 var catToken = n as CatToken;
49 var catToken = n as CatToken;
50 if (catToken != null)
50 if (catToken != null)
51 return Nullable(catToken.Left) && Nullable(catToken.Right);
51 return Nullable(catToken.Left) && Nullable(catToken.Right);
52 return false;
52 return false;
53 }
53 }
54
54
55 protected int Index {
55 protected int Index {
56 get { return m_idx; }
56 get { return m_idx; }
57 }
57 }
58
58
59 public void Visit(AltToken token) {
59 public void Visit(AltToken token) {
60 if (m_root == null)
60 if (m_root == null)
61 m_root = token;
61 m_root = token;
62 var firtspos = new HashSet<int>();
62 var firtspos = new HashSet<int>();
63 var lastpos = new HashSet<int>();
63 var lastpos = new HashSet<int>();
64
64
65 token.Left.Accept(this);
65 token.Left.Accept(this);
66 firtspos.UnionWith(m_firstpos);
66 firtspos.UnionWith(m_firstpos);
67 lastpos.UnionWith(m_lastpos);
67 lastpos.UnionWith(m_lastpos);
68
68
69 token.Right.Accept(this);
69 token.Right.Accept(this);
70 firtspos.UnionWith(m_firstpos);
70 firtspos.UnionWith(m_firstpos);
71 lastpos.UnionWith(m_lastpos);
71 lastpos.UnionWith(m_lastpos);
72
72
73 m_firstpos = firtspos;
73 m_firstpos = firtspos;
74 m_lastpos = lastpos;
74 m_lastpos = lastpos;
75 }
75 }
76
76
77 public void Visit(StarToken token) {
77 public void Visit(StarToken token) {
78 if (m_root == null)
78 if (m_root == null)
79 m_root = token;
79 m_root = token;
80 token.Token.Accept(this);
80 token.Token.Accept(this);
81
81
82 foreach (var i in m_lastpos)
82 foreach (var i in m_lastpos)
83 Followpos(i).UnionWith(m_firstpos);
83 Followpos(i).UnionWith(m_firstpos);
84 }
84 }
85
85
86 public void Visit(CatToken token) {
86 public void Visit(CatToken token) {
87 if (m_root == null)
87 if (m_root == null)
88 m_root = token;
88 m_root = token;
89
89
90 var firtspos = new HashSet<int>();
90 var firtspos = new HashSet<int>();
91 var lastpos = new HashSet<int>();
91 var lastpos = new HashSet<int>();
92 token.Left.Accept(this);
92 token.Left.Accept(this);
93 firtspos.UnionWith(m_firstpos);
93 firtspos.UnionWith(m_firstpos);
94 var leftLastpos = m_lastpos;
94 var leftLastpos = m_lastpos;
95
95
96 token.Right.Accept(this);
96 token.Right.Accept(this);
97 lastpos.UnionWith(m_lastpos);
97 lastpos.UnionWith(m_lastpos);
98 var rightFirstpos = m_firstpos;
98 var rightFirstpos = m_firstpos;
99
99
100 if (Nullable(token.Left))
100 if (Nullable(token.Left))
101 firtspos.UnionWith(rightFirstpos);
101 firtspos.UnionWith(rightFirstpos);
102
102
103 if (Nullable(token.Right))
103 if (Nullable(token.Right))
104 lastpos.UnionWith(leftLastpos);
104 lastpos.UnionWith(leftLastpos);
105
105
106 m_firstpos = firtspos;
106 m_firstpos = firtspos;
107 m_lastpos = lastpos;
107 m_lastpos = lastpos;
108
108
109 foreach (var i in leftLastpos)
109 foreach (var i in leftLastpos)
110 Followpos(i).UnionWith(rightFirstpos);
110 Followpos(i).UnionWith(rightFirstpos);
111
111
112 }
112 }
113
113
114 public void Visit(EmptyToken token) {
114 public void Visit(EmptyToken token) {
115 if (m_root == null)
115 if (m_root == null)
116 m_root = token;
116 m_root = token;
117 }
117 }
118
118
119 public void Visit(SymbolToken token) {
119 public void Visit(SymbolToken token) {
120 if (m_root == null)
120 if (m_root == null)
121 m_root = token;
121 m_root = token;
122 m_idx++;
122 m_idx++;
123 m_indexes[m_idx] = token.Value;
123 m_indexes[m_idx] = token.Value;
124 m_firstpos = new HashSet<int>(new[] { m_idx });
124 m_firstpos = new HashSet<int>(new[] { m_idx });
125 m_lastpos = new HashSet<int>(new[] { m_idx });
125 m_lastpos = new HashSet<int>(new[] { m_idx });
126 }
126 }
127
127
128 public virtual void Visit(EndToken token) {
128 public virtual void Visit(EndToken token) {
129 if (m_root == null)
129 if (m_root == null)
130 m_root = token;
130 m_root = token;
131 m_idx++;
131 m_idx++;
132 m_indexes[m_idx] = AutomatonConst.UNCLASSIFIED_INPUT;
132 m_indexes[m_idx] = AutomatonConst.UnclassifiedInput;
133 m_firstpos = new HashSet<int>(new[] { m_idx });
133 m_firstpos = new HashSet<int>(new[] { m_idx });
134 m_lastpos = new HashSet<int>(new[] { m_idx });
134 m_lastpos = new HashSet<int>(new[] { m_idx });
135 Followpos(m_idx);
135 Followpos(m_idx);
136 m_ends.Add(m_idx);
136 m_ends.Add(m_idx);
137 }
137 }
138
138
139 public void BuildDFA() {
139 public void BuildDFA() {
140 AddState(m_firstpos);
140 AddState(m_firstpos);
141 SetInitialState(m_firstpos);
141 SetInitialState(m_firstpos);
142
142
143 if(IsFinal(m_firstpos))
143 if(IsFinal(m_firstpos))
144 MarkFinalState(m_firstpos);
144 MarkFinalState(m_firstpos);
145
145
146 var inputMax = m_indexes.Values.Max();
146 var inputMax = m_indexes.Values.Max();
147 var queue = new Queue<HashSet<int>>();
147 var queue = new Queue<HashSet<int>>();
148
148
149 queue.Enqueue(m_firstpos);
149 queue.Enqueue(m_firstpos);
150
150
151 while (queue.Count > 0) {
151 while (queue.Count > 0) {
152 var s1 = queue.Dequeue();
152 var s1 = queue.Dequeue();
153
153
154 for (int a = 0; a <= inputMax; a++) {
154 for (int a = 0; a <= inputMax; a++) {
155 var s2 = new HashSet<int>();
155 var s2 = new HashSet<int>();
156 foreach (var p in s1) {
156 foreach (var p in s1) {
157 if (m_indexes[p] == a) {
157 if (m_indexes[p] == a) {
158 s2.UnionWith(Followpos(p));
158 s2.UnionWith(Followpos(p));
159 }
159 }
160 }
160 }
161 if (s2.Count > 0) {
161 if (s2.Count > 0) {
162 if (!HasState(s2)) {
162 if (!HasState(s2)) {
163 AddState(s2);
163 AddState(s2);
164 if (IsFinal(s2))
164 if (IsFinal(s2))
165 MarkFinalState(s2);
165 MarkFinalState(s2);
166
166
167 queue.Enqueue(s2);
167 queue.Enqueue(s2);
168 }
168 }
169
169
170 DefineTransition(s1, s2, a);
170 DefineTransition(s1, s2, a);
171 }
171 }
172
172
173 }
173 }
174 }
174 }
175 }
175 }
176
176
177 protected bool HasState(HashSet<int> state) {
177 protected bool HasState(HashSet<int> state) {
178 return m_states.Contains(state);
178 return m_states.Contains(state);
179 }
179 }
180
180
181 protected void AddState(HashSet<int> state) {
181 protected void AddState(HashSet<int> state) {
182 Debug.Assert(!HasState(state));
182 Debug.Assert(!HasState(state));
183
183
184 m_states.DefineSymbol(state);
184 m_states.DefineSymbol(state);
185 }
185 }
186
186
187 protected int Translate(HashSet<int> state) {
187 protected int Translate(HashSet<int> state) {
188 Debug.Assert(HasState(state));
188 Debug.Assert(HasState(state));
189
189
190 return m_states.Translate(state);
190 return m_states.Translate(state);
191 }
191 }
192
192
193 protected virtual void SetInitialState(HashSet<int> state) {
193 protected virtual void SetInitialState(HashSet<int> state) {
194 m_builder.SetInitialState(Translate(state));
194 m_builder.SetInitialState(Translate(state));
195 }
195 }
196
196
197 protected virtual void MarkFinalState(HashSet<int> state) {
197 protected virtual void MarkFinalState(HashSet<int> state) {
198 m_builder.MarkFinalState(Translate(state));
198 m_builder.MarkFinalState(Translate(state));
199 }
199 }
200
200
201 protected virtual void DefineTransition(HashSet<int> s1, HashSet<int> s2, int ch) {
201 protected virtual void DefineTransition(HashSet<int> s1, HashSet<int> s2, int ch) {
202
202
203 m_builder.Add(new AutomatonTransition(Translate(s1), Translate(s2), ch));
203 m_builder.Add(new AutomatonTransition(Translate(s1), Translate(s2), ch));
204 }
204 }
205
205
206 bool IsFinal(IEnumerable<int> state) {
206 bool IsFinal(IEnumerable<int> state) {
207 Debug.Assert(state != null);
207 Debug.Assert(state != null);
208 return state.Any(m_ends.Contains);
208 return state.Any(m_ends.Contains);
209 }
209 }
210
210
211 }
211 }
212 }
212 }
@@ -1,42 +1,42
1 using Implab.Automaton;
1 using Implab.Automaton;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5 using System.Runtime.CompilerServices;
5 using System.Runtime.CompilerServices;
6 using System.Text;
6 using System.Text;
7 using System.Threading.Tasks;
7 using System.Threading.Tasks;
8
8
9 namespace Implab.Formats {
9 namespace Implab.Formats {
10 public class CharMap : IAlphabet<char> {
10 public class CharMap : IAlphabet<char> {
11 readonly char m_min;
11 readonly char m_min;
12 readonly char m_max;
12 readonly char m_max;
13 readonly int[] m_map;
13 readonly int[] m_map;
14
14
15 public CharMap(char min, int[] map) {
15 public CharMap(char min, int[] map) {
16 Safe.ArgumentNotNull(map, nameof(map));
16 Safe.ArgumentNotNull(map, nameof(map));
17 Count = map.Max()+1;
17 Count = map.Max()+1;
18 m_min = min;
18 m_min = min;
19 m_map = map;
19 m_map = map;
20 m_max = (char)(min + map.Length);
20 m_max = (char)(min + map.Length);
21 }
21 }
22
22
23 public int Count {
23 public int Count {
24 get; private set;
24 get; private set;
25 }
25 }
26
26
27 public bool Contains(char symbol) {
27 public bool Contains(char symbol) {
28 return symbol >= m_min && symbol <= m_max && m_map[symbol-m_min] != AutomatonConst.UNCLASSIFIED_INPUT;
28 return symbol >= m_min && symbol <= m_max && m_map[symbol-m_min] != AutomatonConst.UnclassifiedInput;
29 }
29 }
30
30
31 public IEnumerable<char> GetSymbols(int cls) {
31 public IEnumerable<char> GetSymbols(int cls) {
32 for (var i = 0; i < m_map.Length; i++)
32 for (var i = 0; i < m_map.Length; i++)
33 if (m_map[i] == cls)
33 if (m_map[i] == cls)
34 yield return (char)(i + m_min);
34 yield return (char)(i + m_min);
35 }
35 }
36
36
37 [MethodImpl(MethodImplOptions.AggressiveInlining)]
37 [MethodImpl(MethodImplOptions.AggressiveInlining)]
38 public int Translate(char symbol) {
38 public int Translate(char symbol) {
39 return symbol >= m_min && symbol <= m_max ? m_map[symbol-m_min] : AutomatonConst.UNCLASSIFIED_INPUT;
39 return symbol >= m_min && symbol <= m_max ? m_map[symbol-m_min] : AutomatonConst.UnclassifiedInput;
40 }
40 }
41 }
41 }
42 }
42 }
@@ -1,73 +1,73
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5 using Implab.Automaton;
5 using Implab.Automaton;
6 using Implab.Automaton.RegularExpressions;
6 using Implab.Automaton.RegularExpressions;
7
7
8 namespace Implab.Formats {
8 namespace Implab.Formats {
9 /// <summary>
9 /// <summary>
10 /// Π‘Π°Π·ΠΎΠ²Ρ‹ΠΉ абстрактный класс. Π“Ρ€Π°ΠΌΠΌΠ°Ρ‚ΠΈΠΊΠ°, позволяСт Ρ„ΠΎΡ€ΠΌΡƒΠ»ΠΈΡ€ΠΎΠ²Π°Ρ‚ΡŒ выраТСния Π½Π°Π΄ Π°Π»Ρ„Π°Π²ΠΈΡ‚ΠΎΠΌ Ρ‚ΠΈΠΏΠ° <c>char</c>.
10 /// Π‘Π°Π·ΠΎΠ²Ρ‹ΠΉ абстрактный класс. Π“Ρ€Π°ΠΌΠΌΠ°Ρ‚ΠΈΠΊΠ°, позволяСт Ρ„ΠΎΡ€ΠΌΡƒΠ»ΠΈΡ€ΠΎΠ²Π°Ρ‚ΡŒ выраТСния Π½Π°Π΄ Π°Π»Ρ„Π°Π²ΠΈΡ‚ΠΎΠΌ Ρ‚ΠΈΠΏΠ° <c>char</c>.
11 /// </summary>
11 /// </summary>
12 public abstract class Grammar<TSymbol> {
12 public abstract class Grammar<TSymbol> {
13
13
14 protected abstract IAlphabetBuilder<TSymbol> AlphabetBuilder {
14 protected abstract IAlphabetBuilder<TSymbol> AlphabetBuilder {
15 get;
15 get;
16 }
16 }
17
17
18 protected SymbolToken UnclassifiedToken() {
18 protected SymbolToken UnclassifiedToken() {
19 return new SymbolToken(AutomatonConst.UNCLASSIFIED_INPUT);
19 return new SymbolToken(AutomatonConst.UnclassifiedInput);
20 }
20 }
21
21
22 protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
22 protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
23 Safe.ArgumentNotNull(alphabet, "alphabet");
23 Safe.ArgumentNotNull(alphabet, "alphabet");
24
24
25 foreach (var ch in alphabet)
25 foreach (var ch in alphabet)
26 AlphabetBuilder.DefineSymbol(ch);
26 AlphabetBuilder.DefineSymbol(ch);
27 }
27 }
28
28
29 protected Token SymbolToken(TSymbol symbol) {
29 protected Token SymbolToken(TSymbol symbol) {
30 return Token.New(TranslateOrAdd(symbol));
30 return Token.New(TranslateOrAdd(symbol));
31 }
31 }
32
32
33 protected Token SymbolToken(IEnumerable<TSymbol> symbols) {
33 protected Token SymbolToken(IEnumerable<TSymbol> symbols) {
34 Safe.ArgumentNotNull(symbols, "symbols");
34 Safe.ArgumentNotNull(symbols, "symbols");
35
35
36 return Token.New(TranslateOrAdd(symbols).ToArray());
36 return Token.New(TranslateOrAdd(symbols).ToArray());
37 }
37 }
38
38
39 protected Token SymbolSetToken(params TSymbol[] set) {
39 protected Token SymbolSetToken(params TSymbol[] set) {
40 return SymbolToken(set);
40 return SymbolToken(set);
41 }
41 }
42
42
43 int TranslateOrAdd(TSymbol ch) {
43 int TranslateOrAdd(TSymbol ch) {
44 var t = AlphabetBuilder.Translate(ch);
44 var t = AlphabetBuilder.Translate(ch);
45 if (t == AutomatonConst.UNCLASSIFIED_INPUT)
45 if (t == AutomatonConst.UnclassifiedInput)
46 t = AlphabetBuilder.DefineSymbol(ch);
46 t = AlphabetBuilder.DefineSymbol(ch);
47 return t;
47 return t;
48 }
48 }
49
49
50 IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
50 IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
51 return symbols.Distinct().Select(TranslateOrAdd);
51 return symbols.Distinct().Select(TranslateOrAdd);
52 }
52 }
53
53
54 int TranslateOrDie(TSymbol ch) {
54 int TranslateOrDie(TSymbol ch) {
55 var t = AlphabetBuilder.Translate(ch);
55 var t = AlphabetBuilder.Translate(ch);
56 if (t == AutomatonConst.UNCLASSIFIED_INPUT)
56 if (t == AutomatonConst.UnclassifiedInput)
57 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
57 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
58 return t;
58 return t;
59 }
59 }
60
60
61 IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
61 IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
62 return symbols.Distinct().Select(TranslateOrDie);
62 return symbols.Distinct().Select(TranslateOrDie);
63 }
63 }
64
64
65 protected Token SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
65 protected Token SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
66 Safe.ArgumentNotNull(symbols, "symbols");
66 Safe.ArgumentNotNull(symbols, "symbols");
67
67
68 return Token.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
68 return Token.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
69 }
69 }
70 }
70 }
71
71
72
72
73 }
73 }
@@ -1,84 +1,84
1 using Implab.Automaton;
1 using Implab.Automaton;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5 using System.Runtime.CompilerServices;
5 using System.Runtime.CompilerServices;
6 using System.Text;
6 using System.Text;
7 using System.Threading.Tasks;
7 using System.Threading.Tasks;
8
8
9 namespace Implab.Formats {
9 namespace Implab.Formats {
10 public class InputScanner<TTag> {
10 public class InputScanner<TTag> {
11 readonly TTag[] m_tags;
11 readonly TTag[] m_tags;
12 readonly int m_initialState;
12 readonly int m_initialState;
13 readonly int[,] m_dfa;
13 readonly int[,] m_dfa;
14 readonly CharMap m_alphabet;
14 readonly CharMap m_alphabet;
15 readonly bool[] m_final;
15 readonly bool[] m_final;
16
16
17 int m_position;
17 int m_position;
18 int m_state;
18 int m_state;
19
19
20 public InputScanner(int[,] dfaTable, bool[] finalStates, TTag[] tags, int initialState, CharMap alphabet) {
20 public InputScanner(int[,] dfaTable, bool[] finalStates, TTag[] tags, int initialState, CharMap alphabet) {
21 Safe.ArgumentNotNull(dfaTable, nameof(dfaTable));
21 Safe.ArgumentNotNull(dfaTable, nameof(dfaTable));
22 Safe.ArgumentNotNull(finalStates, nameof(finalStates));
22 Safe.ArgumentNotNull(finalStates, nameof(finalStates));
23 Safe.ArgumentNotNull(tags, nameof(tags));
23 Safe.ArgumentNotNull(tags, nameof(tags));
24 Safe.ArgumentNotNull(alphabet, nameof(alphabet));
24 Safe.ArgumentNotNull(alphabet, nameof(alphabet));
25
25
26 m_dfa = dfaTable;
26 m_dfa = dfaTable;
27 m_final = finalStates;
27 m_final = finalStates;
28 m_tags = tags;
28 m_tags = tags;
29 m_initialState = initialState;
29 m_initialState = initialState;
30 m_alphabet = alphabet;
30 m_alphabet = alphabet;
31 }
31 }
32
32
33 public TTag Tag {
33 public TTag Tag {
34 [MethodImpl(MethodImplOptions.AggressiveInlining)]
34 [MethodImpl(MethodImplOptions.AggressiveInlining)]
35 get {
35 get {
36 return m_tags[m_state];
36 return m_tags[m_state];
37 }
37 }
38 }
38 }
39
39
40 public int Position {
40 public int Position {
41 [MethodImpl(MethodImplOptions.AggressiveInlining)]
41 [MethodImpl(MethodImplOptions.AggressiveInlining)]
42 get {
42 get {
43 return m_position;
43 return m_position;
44 }
44 }
45 }
45 }
46
46
47 public bool IsFinal {
47 public bool IsFinal {
48 [MethodImpl(MethodImplOptions.AggressiveInlining)]
48 [MethodImpl(MethodImplOptions.AggressiveInlining)]
49 get {
49 get {
50 return m_final[m_state];
50 return m_final[m_state];
51 }
51 }
52 }
52 }
53
53
54 [MethodImpl(MethodImplOptions.AggressiveInlining)]
54 [MethodImpl(MethodImplOptions.AggressiveInlining)]
55 public void ResetState() {
55 public void ResetState() {
56 m_state = m_initialState;
56 m_state = m_initialState;
57 }
57 }
58
58
59 public InputScanner<TTag> Clone() {
59 public InputScanner<TTag> Clone() {
60 var clone = new InputScanner<TTag>(m_dfa, m_final, m_tags, m_initialState, m_alphabet);
60 var clone = new InputScanner<TTag>(m_dfa, m_final, m_tags, m_initialState, m_alphabet);
61 clone.m_state = m_state;
61 clone.m_state = m_state;
62 clone.m_position = m_position;
62 clone.m_position = m_position;
63 return clone;
63 return clone;
64 }
64 }
65
65
66 //[MethodImpl(MethodImplOptions.AggressiveInlining)]
66 //[MethodImpl(MethodImplOptions.AggressiveInlining)]
67 public bool Scan(char[] data, int offset, int max) {
67 public bool Scan(char[] data, int offset, int max) {
68 var next = m_state;
68 var next = m_state;
69
69
70 while(offset < max) {
70 while(offset < max) {
71 next = m_dfa[next, m_alphabet.Translate(data[offset])];
71 next = m_dfa[next, m_alphabet.Translate(data[offset])];
72 if (next == AutomatonConst.UNREACHABLE_STATE) {
72 if (next == AutomatonConst.UnreachableState) {
73 // scanner stops on the next position after last recognized symbol
73 // scanner stops on the next position after last recognized symbol
74 m_position = offset;
74 m_position = offset;
75 return false;
75 return false;
76 }
76 }
77 m_state = next;
77 m_state = next;
78 offset++;
78 offset++;
79 }
79 }
80 m_position = offset;
80 m_position = offset;
81 return true;
81 return true;
82 }
82 }
83 }
83 }
84 }
84 }
@@ -1,148 +1,148
1 using System.Linq;
1 using System.Linq;
2 using Implab.Automaton.RegularExpressions;
2 using Implab.Automaton.RegularExpressions;
3 using System;
3 using System;
4 using Implab.Automaton;
4 using Implab.Automaton;
5 using Implab.Components;
5 using Implab.Components;
6
6
7 namespace Implab.Formats.Json {
7 namespace Implab.Formats.Json {
8 public class JsonGrammar : Grammar<char> {
8 public class JsonGrammar : Grammar<char> {
9 public enum TokenType {
9 public enum TokenType {
10 None,
10 None,
11 BeginObject,
11 BeginObject,
12 EndObject,
12 EndObject,
13 BeginArray,
13 BeginArray,
14 EndArray,
14 EndArray,
15 String,
15 String,
16 Number,
16 Number,
17 Literal,
17 Literal,
18 NameSeparator,
18 NameSeparator,
19 ValueSeparator,
19 ValueSeparator,
20 Whitespace,
20 Whitespace,
21
21
22 StringBound,
22 StringBound,
23 EscapedChar,
23 EscapedChar,
24 UnescapedChar,
24 UnescapedChar,
25 EscapedUnicode
25 EscapedUnicode
26 }
26 }
27
27
28 static LazyAndWeak<JsonGrammar> _instance = new LazyAndWeak<JsonGrammar>(() => new JsonGrammar());
28 static LazyAndWeak<JsonGrammar> _instance = new LazyAndWeak<JsonGrammar>(() => new JsonGrammar());
29
29
30 public static JsonGrammar Instance {
30 public static JsonGrammar Instance {
31 get { return _instance.Value; }
31 get { return _instance.Value; }
32 }
32 }
33
33
34 readonly InputScanner<TokenType> m_jsonExpression;
34 readonly FastInputScanner<TokenType> m_jsonExpression;
35 readonly InputScanner<TokenType> m_stringExpression;
35 readonly FastInputScanner<TokenType> m_stringExpression;
36 readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
36 readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
37
37
38 public CharAlphabet DefaultAlphabet { get { return m_defaultAlphabet; } }
38 public CharAlphabet DefaultAlphabet { get { return m_defaultAlphabet; } }
39
39
40 public JsonGrammar() {
40 public JsonGrammar() {
41 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
41 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
42 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
42 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
43 var digit9 = SymbolRangeToken('1', '9');
43 var digit9 = SymbolRangeToken('1', '9');
44 var zero = SymbolToken('0');
44 var zero = SymbolToken('0');
45 var digit = zero.Or(digit9);
45 var digit = zero.Or(digit9);
46 var dot = SymbolToken('.');
46 var dot = SymbolToken('.');
47 var minus = SymbolToken('-');
47 var minus = SymbolToken('-');
48 var sign = SymbolSetToken('-', '+');
48 var sign = SymbolSetToken('-', '+');
49 var expSign = SymbolSetToken('e', 'E');
49 var expSign = SymbolSetToken('e', 'E');
50 var letters = SymbolRangeToken('a', 'z');
50 var letters = SymbolRangeToken('a', 'z');
51 var integer = zero.Or(digit9.Cat(digit.EClosure()));
51 var integer = zero.Or(digit9.Cat(digit.EClosure()));
52 var frac = dot.Cat(digit.Closure());
52 var frac = dot.Cat(digit.Closure());
53 var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
53 var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
54 var quote = SymbolToken('"');
54 var quote = SymbolToken('"');
55 var backSlash = SymbolToken('\\');
55 var backSlash = SymbolToken('\\');
56 var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
56 var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
57 var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
57 var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
58 var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
58 var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
59 var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
59 var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
60 var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
60 var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
61 var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
61 var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
62 var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
62 var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
63 var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
63 var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
64 var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
64 var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
65
65
66 var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
66 var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
67 var literal = letters.Closure();
67 var literal = letters.Closure();
68 var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
68 var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
69
69
70 var jsonExpression =
70 var jsonExpression =
71 number.Tag(TokenType.Number)
71 number.Tag(TokenType.Number)
72 .Or(literal.Tag(TokenType.Literal))
72 .Or(literal.Tag(TokenType.Literal))
73 .Or(quote.Tag(TokenType.StringBound))
73 .Or(quote.Tag(TokenType.StringBound))
74 .Or(beginObject.Tag(TokenType.BeginObject))
74 .Or(beginObject.Tag(TokenType.BeginObject))
75 .Or(endObject.Tag(TokenType.EndObject))
75 .Or(endObject.Tag(TokenType.EndObject))
76 .Or(beginArray.Tag(TokenType.BeginArray))
76 .Or(beginArray.Tag(TokenType.BeginArray))
77 .Or(endArray.Tag(TokenType.EndArray))
77 .Or(endArray.Tag(TokenType.EndArray))
78 .Or(nameSep.Tag(TokenType.NameSeparator))
78 .Or(nameSep.Tag(TokenType.NameSeparator))
79 .Or(valueSep.Tag(TokenType.ValueSeparator))
79 .Or(valueSep.Tag(TokenType.ValueSeparator))
80 .Or(SymbolSetToken('\n', '\r', '\t', ' ').Closure().Tag(TokenType.Whitespace));
80 .Or(SymbolSetToken('\n', '\r', '\t', ' ').Closure().Tag(TokenType.Whitespace));
81
81
82
82
83 var jsonStringExpression =
83 var jsonStringExpression =
84 quote.Tag(TokenType.StringBound)
84 quote.Tag(TokenType.StringBound)
85 .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
85 .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
86 .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
86 .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
87 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
87 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
88
88
89
89
90 m_jsonExpression = BuildScanner(jsonExpression);
90 m_jsonExpression = BuildFastScanner(jsonExpression);
91 m_stringExpression = BuildScanner(jsonStringExpression);
91 m_stringExpression = BuildFastScanner(jsonStringExpression);
92 }
92 }
93
93
94 public static InputScanner<TokenType> CreateJsonExpressionScanner() {
94 public static FastInputScanner<TokenType> CreateJsonExpressionScanner() {
95 return Instance.m_jsonExpression.Clone();
95 return Instance.m_jsonExpression.Clone();
96 }
96 }
97
97
98 public static InputScanner<TokenType> CreateStringExpressionScanner() {
98 public static FastInputScanner<TokenType> CreateStringExpressionScanner() {
99 return Instance.m_stringExpression.Clone();
99 return Instance.m_stringExpression.Clone();
100 }
100 }
101
101
102 protected override IAlphabetBuilder<char> AlphabetBuilder {
102 protected override IAlphabetBuilder<char> AlphabetBuilder {
103 get {
103 get {
104 return m_defaultAlphabet;
104 return m_defaultAlphabet;
105 }
105 }
106 }
106 }
107
107
108 Token SymbolRangeToken(char start, char stop) {
108 Token SymbolRangeToken(char start, char stop) {
109 return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
109 return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
110 }
110 }
111
111
112 public InputScanner<TokenType> BuildScanner(Token regexp) {
112 public FastInputScanner<TokenType> BuildFastScanner(Token regexp) {
113 var dfa = new RegularDFA<char, TokenType>(AlphabetBuilder);
113 var dfa = new RegularDFA<char, TokenType>(AlphabetBuilder);
114
114
115 var visitor = new RegularExpressionVisitor<TokenType>(dfa);
115 var visitor = new RegularExpressionVisitor<TokenType>(dfa);
116 regexp.Accept(visitor);
116 regexp.Accept(visitor);
117 visitor.BuildDFA();
117 visitor.BuildDFA();
118
118
119 if (dfa.IsFinalState(dfa.InitialState))
119 if (dfa.IsFinalState(dfa.InitialState))
120 throw new ApplicationException("The specified language contains empty token");
120 throw new ApplicationException("The specified language contains empty token");
121
121
122 var ab = new CharAlphabet();
122 var ab = new CharAlphabet();
123 var optimal = dfa.Optimize(ab);
123 var optimal = dfa.Optimize(ab);
124
124
125 return new InputScanner<TokenType>(
125 return new FastInputScanner<TokenType>(
126 optimal.CreateTransitionTable(),
126 optimal.CreateTransitionTable(),
127 optimal.CreateFinalStateTable(),
127 optimal.CreateFinalStateTable(),
128 NormalizeTags(optimal.CreateTagTable()),
128 NormalizeTags(optimal.CreateTagTable()),
129 optimal.InitialState,
129 optimal.InitialState,
130 ab.CreateCharMap()
130 ab.GetTranslationMap()
131 );
131 );
132 }
132 }
133
133
134 static TokenType[] NormalizeTags(TokenType[][] tags) {
134 static TokenType[] NormalizeTags(TokenType[][] tags) {
135 var result = new TokenType[tags.Length];
135 var result = new TokenType[tags.Length];
136 for(var i = 0; i< tags.Length; i++) {
136 for(var i = 0; i< tags.Length; i++) {
137 if (tags[i] == null || tags[i].Length == 0)
137 if (tags[i] == null || tags[i].Length == 0)
138 result[i] = default(TokenType);
138 result[i] = default(TokenType);
139 else if (tags[i].Length == 1)
139 else if (tags[i].Length == 1)
140 result[i] = tags[i][0];
140 result[i] = tags[i][0];
141 else
141 else
142 throw new Exception($"Ambigous state tags {string.Join(", ", tags[i])}");
142 throw new Exception($"Ambigous state tags {string.Join(", ", tags[i])}");
143 }
143 }
144 return result;
144 return result;
145 }
145 }
146
146
147 }
147 }
148 }
148 }
@@ -1,318 +1,318
1 using System;
1 using System;
2 using System.Diagnostics;
2 using System.Diagnostics;
3 using System.IO;
3 using System.IO;
4 using Implab.Automaton;
4 using Implab.Automaton;
5 using Implab.Automaton.RegularExpressions;
5 using Implab.Automaton.RegularExpressions;
6 using System.Linq;
6 using System.Linq;
7 using Implab.Components;
7 using Implab.Components;
8 using System.Collections.Generic;
8 using System.Collections.Generic;
9 using System.Text;
9 using System.Text;
10 using System.Globalization;
10 using System.Globalization;
11
11
12 namespace Implab.Formats.Json {
12 namespace Implab.Formats.Json {
13 /// <summary>
13 /// <summary>
14 /// Pull парсСр JSON Π΄Π°Π½Π½Ρ‹Ρ….
14 /// Pull парсСр JSON Π΄Π°Π½Π½Ρ‹Ρ….
15 /// </summary>
15 /// </summary>
16 /// <remarks>
16 /// <remarks>
17 /// Π‘Π»Π΅Π΄ΡƒΠ΅Ρ‚ ΠΎΡ‚ΠΌΠ΅Ρ‚ΠΈΡ‚ΡŒ ΠΎΡ‚Π΄Π΅Π»ΡŒΠ½ΡƒΡŽ ΠΈΠ½Ρ‚Π΅Ρ€ΠΏΡ€Π΅Ρ‚Π°Ρ†ΠΈΡŽ свойства <see cref="Level"/>,
17 /// Π‘Π»Π΅Π΄ΡƒΠ΅Ρ‚ ΠΎΡ‚ΠΌΠ΅Ρ‚ΠΈΡ‚ΡŒ ΠΎΡ‚Π΄Π΅Π»ΡŒΠ½ΡƒΡŽ ΠΈΠ½Ρ‚Π΅Ρ€ΠΏΡ€Π΅Ρ‚Π°Ρ†ΠΈΡŽ свойства <see cref="Level"/>,
18 /// ΠΎΠ½ΠΎ ΠΎΠ·Π½Π°Ρ‡Π°Π΅Ρ‚ Ρ‚Π΅ΠΊΡƒΡ‰ΠΈΠΉ ΡƒΡ€ΠΎΠ²Π΅Π½ΡŒ влоТСнности ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ΠΎΠ², ΠΎΠ΄Π½Π°ΠΊΠΎ Π·Π°ΠΊΡ€Ρ‹Π²Π°ΡŽΡ‰ΠΈΠΉ
18 /// ΠΎΠ½ΠΎ ΠΎΠ·Π½Π°Ρ‡Π°Π΅Ρ‚ Ρ‚Π΅ΠΊΡƒΡ‰ΠΈΠΉ ΡƒΡ€ΠΎΠ²Π΅Π½ΡŒ влоТСнности ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ΠΎΠ², ΠΎΠ΄Π½Π°ΠΊΠΎ Π·Π°ΠΊΡ€Ρ‹Π²Π°ΡŽΡ‰ΠΈΠΉ
19 /// элСмСнт ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Π° ΠΈ массива ΠΈΠΌΠ΅Π΅Ρ‚ ΡƒΡ€ΠΎΠ²Π΅Π½ΡŒ мСньшС, Ρ‡Π΅ΠΌ сам ΠΎΠ±ΡŠΠ΅ΠΊΡ‚.
19 /// элСмСнт ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Π° ΠΈ массива ΠΈΠΌΠ΅Π΅Ρ‚ ΡƒΡ€ΠΎΠ²Π΅Π½ΡŒ мСньшС, Ρ‡Π΅ΠΌ сам ΠΎΠ±ΡŠΠ΅ΠΊΡ‚.
20 /// <code>
20 /// <code>
21 /// { // Level = 1
21 /// { // Level = 1
22 /// "name" : "Peter", // Level = 1
22 /// "name" : "Peter", // Level = 1
23 /// "address" : { // Level = 2
23 /// "address" : { // Level = 2
24 /// city : "Stern" // Level = 2
24 /// city : "Stern" // Level = 2
25 /// } // Level = 1
25 /// } // Level = 1
26 /// } // Level = 0
26 /// } // Level = 0
27 /// </code>
27 /// </code>
28 /// </remarks>
28 /// </remarks>
29 public class JsonReader : Disposable {
29 public class JsonReader : Disposable {
30
30
31 enum MemberContext {
31 enum MemberContext {
32 MemberName,
32 MemberName,
33 MemberValue
33 MemberValue
34 }
34 }
35
35
36 #region Parser rules
36 #region Parser rules
37 struct ParserContext {
37 struct ParserContext {
38 readonly int[,] m_dfa;
38 readonly int[,] m_dfa;
39 int m_state;
39 int m_state;
40
40
41 readonly JsonElementContext m_elementContext;
41 readonly JsonElementContext m_elementContext;
42
42
43 public ParserContext(int[,] dfa, int state, JsonElementContext context) {
43 public ParserContext(int[,] dfa, int state, JsonElementContext context) {
44 m_dfa = dfa;
44 m_dfa = dfa;
45 m_state = state;
45 m_state = state;
46 m_elementContext = context;
46 m_elementContext = context;
47 }
47 }
48
48
49 public bool Move(JsonTokenType token) {
49 public bool Move(JsonTokenType token) {
50 var next = m_dfa[m_state, (int)token];
50 var next = m_dfa[m_state, (int)token];
51 if (next == AutomatonConst.UNREACHABLE_STATE)
51 if (next == AutomatonConst.UnreachableState)
52 return false;
52 return false;
53 m_state = next;
53 m_state = next;
54 return true;
54 return true;
55 }
55 }
56
56
57 public JsonElementContext ElementContext {
57 public JsonElementContext ElementContext {
58 get { return m_elementContext; }
58 get { return m_elementContext; }
59 }
59 }
60 }
60 }
61
61
62 static readonly ParserContext _jsonContext;
62 static readonly ParserContext _jsonContext;
63 static readonly ParserContext _objectContext;
63 static readonly ParserContext _objectContext;
64 static readonly ParserContext _arrayContext;
64 static readonly ParserContext _arrayContext;
65
65
66 static JsonReader() {
66 static JsonReader() {
67
67
68 var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
68 var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
69 var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
69 var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
70
70
71 var objectExpression = memberExpression
71 var objectExpression = memberExpression
72 .Cat(
72 .Cat(
73 MakeToken(JsonTokenType.ValueSeparator)
73 MakeToken(JsonTokenType.ValueSeparator)
74 .Cat(memberExpression)
74 .Cat(memberExpression)
75 .EClosure()
75 .EClosure()
76 )
76 )
77 .Optional()
77 .Optional()
78 .Cat(MakeToken(JsonTokenType.EndObject))
78 .Cat(MakeToken(JsonTokenType.EndObject))
79 .End();
79 .End();
80
80
81 var arrayExpression = valueExpression
81 var arrayExpression = valueExpression
82 .Cat(
82 .Cat(
83 MakeToken(JsonTokenType.ValueSeparator)
83 MakeToken(JsonTokenType.ValueSeparator)
84 .Cat(valueExpression)
84 .Cat(valueExpression)
85 .EClosure()
85 .EClosure()
86 )
86 )
87 .Optional()
87 .Optional()
88 .Cat(MakeToken(JsonTokenType.EndArray))
88 .Cat(MakeToken(JsonTokenType.EndArray))
89 .End();
89 .End();
90
90
91 var jsonExpression = valueExpression.End();
91 var jsonExpression = valueExpression.End();
92
92
93 _jsonContext = CreateParserContext(jsonExpression, JsonElementContext.None);
93 _jsonContext = CreateParserContext(jsonExpression, JsonElementContext.None);
94 _objectContext = CreateParserContext(objectExpression, JsonElementContext.Object);
94 _objectContext = CreateParserContext(objectExpression, JsonElementContext.Object);
95 _arrayContext = CreateParserContext(arrayExpression, JsonElementContext.Array);
95 _arrayContext = CreateParserContext(arrayExpression, JsonElementContext.Array);
96 }
96 }
97
97
98 static Token MakeToken(params JsonTokenType[] input) {
98 static Token MakeToken(params JsonTokenType[] input) {
99 return Token.New( input.Select(t => (int)t).ToArray() );
99 return Token.New( input.Select(t => (int)t).ToArray() );
100 }
100 }
101
101
102 static ParserContext CreateParserContext(Token expr, JsonElementContext context) {
102 static ParserContext CreateParserContext(Token expr, JsonElementContext context) {
103
103
104 var dfa = new DFATable();
104 var dfa = new DFATable();
105 var builder = new RegularExpressionVisitor(dfa);
105 var builder = new RegularExpressionVisitor(dfa);
106 expr.Accept(builder);
106 expr.Accept(builder);
107 builder.BuildDFA();
107 builder.BuildDFA();
108
108
109 return new ParserContext(dfa.CreateTransitionTable(), dfa.InitialState, context);
109 return new ParserContext(dfa.CreateTransitionTable(), dfa.InitialState, context);
110 }
110 }
111
111
112 #endregion
112 #endregion
113
113
114 readonly JsonScanner m_scanner;
114 readonly JsonScanner m_scanner;
115 // json starts from the value context and may content even a single literal
115 // json starts from the value context and may content even a single literal
116 MemberContext m_memberContext = MemberContext.MemberValue;
116 MemberContext m_memberContext = MemberContext.MemberValue;
117
117
118 JsonElementType m_elementType;
118 JsonElementType m_elementType;
119 object m_elementValue;
119 string m_elementValue;
120 string m_memberName = String.Empty;
120 string m_memberName = String.Empty;
121
121
122 Stack<ParserContext> m_stack = new Stack<ParserContext>();
122 Stack<ParserContext> m_stack = new Stack<ParserContext>();
123 ParserContext m_context = _jsonContext;
123 ParserContext m_context = _jsonContext;
124
124
125 /// <summary>
125 /// <summary>
126 /// Π‘ΠΎΠ·Π΄Π°Π΅Ρ‚ Π½ΠΎΠ²Ρ‹ΠΉ парсСр Π½Π° основС строки, содСрТащСй JSON
126 /// Π‘ΠΎΠ·Π΄Π°Π΅Ρ‚ Π½ΠΎΠ²Ρ‹ΠΉ парсСр Π½Π° основС строки, содСрТащСй JSON
127 /// </summary>
127 /// </summary>
128 /// <param name="text"></param>
128 /// <param name="text"></param>
129 JsonReader(JsonScanner scanner) {
129 JsonReader(JsonScanner scanner) {
130 m_scanner = scanner;
130 m_scanner = scanner;
131 }
131 }
132
132
133 public int Level {
133 public int Level {
134 get { return m_stack.Count; }
134 get { return m_stack.Count; }
135 }
135 }
136
136
137 /// <summary>
137 /// <summary>
138 /// Π’ΠΈΠΏ Ρ‚Π΅ΠΊΡƒΡ‰Π΅Π³ΠΎ элСмСнта Π½Π° ΠΊΠΎΡ‚ΠΎΡ€ΠΎΠΌ стоит парсСр.
138 /// Π’ΠΈΠΏ Ρ‚Π΅ΠΊΡƒΡ‰Π΅Π³ΠΎ элСмСнта Π½Π° ΠΊΠΎΡ‚ΠΎΡ€ΠΎΠΌ стоит парсСр.
139 /// </summary>
139 /// </summary>
140 public JsonElementType ElementType {
140 public JsonElementType ElementType {
141 get { return m_elementType; }
141 get { return m_elementType; }
142 }
142 }
143
143
144 /// <summary>
144 /// <summary>
145 /// Имя элСмСнта - имя свойства Ρ€ΠΎΠ΄ΠΈΡ‚Π΅Π»ΡŒΡΠΊΠΎΠ³ΠΎ ΠΊΠΎΠ½Ρ‚Π΅ΠΉΠ½Π΅Ρ€Π°. Для элСмСнтов массивов ΠΈ ΠΊΠΎΡ€Π½Π΅Π²ΠΎΠ³ΠΎ всСгда
145 /// Имя элСмСнта - имя свойства Ρ€ΠΎΠ΄ΠΈΡ‚Π΅Π»ΡŒΡΠΊΠΎΠ³ΠΎ ΠΊΠΎΠ½Ρ‚Π΅ΠΉΠ½Π΅Ρ€Π°. Для элСмСнтов массивов ΠΈ ΠΊΠΎΡ€Π½Π΅Π²ΠΎΠ³ΠΎ всСгда
146 /// пустая строка.
146 /// пустая строка.
147 /// </summary>
147 /// </summary>
148 public string ElementName {
148 public string ElementName {
149 get { return m_memberName; }
149 get { return m_memberName; }
150 }
150 }
151
151
152 /// <summary>
152 /// <summary>
153 /// Π—Π½Π°Ρ‡Π΅Π½ΠΈΠ΅ элСмСнта. Волько для элСмСнтов Ρ‚ΠΈΠΏΠ° <see cref="JsonElementType.Value"/>, для ΠΎΡΡ‚Π°Π»ΡŒΠ½Ρ‹Ρ… <c>null</c>
153 /// Π—Π½Π°Ρ‡Π΅Π½ΠΈΠ΅ элСмСнта. Волько для элСмСнтов Ρ‚ΠΈΠΏΠ° <see cref="JsonElementType.Value"/>, для ΠΎΡΡ‚Π°Π»ΡŒΠ½Ρ‹Ρ… <c>null</c>
154 /// </summary>
154 /// </summary>
155 public object ElementValue {
155 public string ElementValue {
156 get { return m_elementValue; }
156 get { return m_elementValue; }
157 }
157 }
158
158
159 /// <summary>
159 /// <summary>
160 /// Π§ΠΈΡ‚Π°Π΅Ρ‚ ΡΠ»Π΅ΡŽΡƒΠ΄ΡƒΡ‰ΠΈΠΉ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ ΠΈΠ· ΠΏΠΎΡ‚ΠΎΠΊΠ°
160 /// Π§ΠΈΡ‚Π°Π΅Ρ‚ ΡΠ»Π΅ΡŽΡƒΠ΄ΡƒΡ‰ΠΈΠΉ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ ΠΈΠ· ΠΏΠΎΡ‚ΠΎΠΊΠ°
161 /// </summary>
161 /// </summary>
162 /// <returns><c>true</c> - опСрация чтСния ΠΏΡ€ΠΎΡˆΠ»Π° ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ, <c>false</c> - ΠΊΠΎΠ½Π΅Ρ† Π΄Π°Π½Π½Ρ‹Ρ…</returns>
162 /// <returns><c>true</c> - опСрация чтСния ΠΏΡ€ΠΎΡˆΠ»Π° ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ, <c>false</c> - ΠΊΠΎΠ½Π΅Ρ† Π΄Π°Π½Π½Ρ‹Ρ…</returns>
163 public bool Read() {
163 public bool Read() {
164 string tokenValue;
164 string tokenValue;
165 JsonTokenType tokenType;
165 JsonTokenType tokenType;
166
166
167 m_memberName = String.Empty;
167 m_memberName = String.Empty;
168
168
169 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
169 while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
170 if(!m_context.Move(tokenType))
170 if(!m_context.Move(tokenType))
171 UnexpectedToken(tokenValue, tokenType);
171 UnexpectedToken(tokenValue, tokenType);
172
172
173 switch (tokenType) {
173 switch (tokenType) {
174 case JsonTokenType.BeginObject:
174 case JsonTokenType.BeginObject:
175 m_stack.Push(m_context);
175 m_stack.Push(m_context);
176 m_context = _objectContext;
176 m_context = _objectContext;
177
177
178 m_elementValue = null;
178 m_elementValue = null;
179 m_memberContext = MemberContext.MemberName;
179 m_memberContext = MemberContext.MemberName;
180 m_elementType = JsonElementType.BeginObject;
180 m_elementType = JsonElementType.BeginObject;
181 return true;
181 return true;
182 case JsonTokenType.EndObject:
182 case JsonTokenType.EndObject:
183 if (m_stack.Count == 0)
183 if (m_stack.Count == 0)
184 UnexpectedToken(tokenValue, tokenType);
184 UnexpectedToken(tokenValue, tokenType);
185 m_context = m_stack.Pop();
185 m_context = m_stack.Pop();
186
186
187 m_elementValue = null;
187 m_elementValue = null;
188 m_elementType = JsonElementType.EndObject;
188 m_elementType = JsonElementType.EndObject;
189 return true;
189 return true;
190 case JsonTokenType.BeginArray:
190 case JsonTokenType.BeginArray:
191 m_stack.Push(m_context);
191 m_stack.Push(m_context);
192 m_context = _arrayContext;
192 m_context = _arrayContext;
193
193
194 m_elementValue = null;
194 m_elementValue = null;
195 m_memberContext = MemberContext.MemberValue;
195 m_memberContext = MemberContext.MemberValue;
196 m_elementType = JsonElementType.BeginArray;
196 m_elementType = JsonElementType.BeginArray;
197 return true;
197 return true;
198 case JsonTokenType.EndArray:
198 case JsonTokenType.EndArray:
199 if (m_stack.Count == 0)
199 if (m_stack.Count == 0)
200 UnexpectedToken(tokenValue, tokenType);
200 UnexpectedToken(tokenValue, tokenType);
201 m_context = m_stack.Pop();
201 m_context = m_stack.Pop();
202
202
203 m_elementValue = null;
203 m_elementValue = null;
204 m_elementType = JsonElementType.EndArray;
204 m_elementType = JsonElementType.EndArray;
205 return true;
205 return true;
206 case JsonTokenType.String:
206 case JsonTokenType.String:
207 if (m_memberContext == MemberContext.MemberName) {
207 if (m_memberContext == MemberContext.MemberName) {
208 m_memberName = tokenValue;
208 m_memberName = tokenValue;
209 break;
209 break;
210 }
210 }
211 m_elementType = JsonElementType.Value;
211 m_elementType = JsonElementType.Value;
212 m_elementValue = tokenValue;
212 m_elementValue = tokenValue;
213 return true;
213 return true;
214 case JsonTokenType.Number:
214 case JsonTokenType.Number:
215 m_elementType = JsonElementType.Value;
215 m_elementType = JsonElementType.Value;
216 m_elementValue = double.Parse(tokenValue, CultureInfo.InvariantCulture);
216 m_elementValue = tokenValue;
217 return true;
217 return true;
218 case JsonTokenType.Literal:
218 case JsonTokenType.Literal:
219 m_elementType = JsonElementType.Value;
219 m_elementType = JsonElementType.Value;
220 m_elementValue = ParseLiteral(tokenValue);
220 m_elementValue = tokenValue == "null" ? null : tokenValue;
221 return true;
221 return true;
222 case JsonTokenType.NameSeparator:
222 case JsonTokenType.NameSeparator:
223 m_memberContext = MemberContext.MemberValue;
223 m_memberContext = MemberContext.MemberValue;
224 break;
224 break;
225 case JsonTokenType.ValueSeparator:
225 case JsonTokenType.ValueSeparator:
226 m_memberContext = m_context.ElementContext == JsonElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
226 m_memberContext = m_context.ElementContext == JsonElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
227 break;
227 break;
228 default:
228 default:
229 UnexpectedToken(tokenValue, tokenType);
229 UnexpectedToken(tokenValue, tokenType);
230 break;
230 break;
231 }
231 }
232 }
232 }
233 if (m_context.ElementContext != JsonElementContext.None)
233 if (m_context.ElementContext != JsonElementContext.None)
234 throw new ParserException("Unexpedted end of data");
234 throw new ParserException("Unexpedted end of data");
235
235
236 Eof = true;
236 Eof = true;
237
237
238 return false;
238 return false;
239 }
239 }
240
240
241 object ParseLiteral(string literal) {
241 object ParseLiteral(string literal) {
242 switch (literal) {
242 switch (literal) {
243 case "null":
243 case "null":
244 return null;
244 return null;
245 case "false":
245 case "false":
246 return false;
246 return false;
247 case "true":
247 case "true":
248 return true;
248 return true;
249 default:
249 default:
250 UnexpectedToken(literal, JsonTokenType.Literal);
250 UnexpectedToken(literal, JsonTokenType.Literal);
251 return null; // avoid compliler error
251 return null; // avoid compliler error
252 }
252 }
253 }
253 }
254
254
255 void UnexpectedToken(object value, JsonTokenType tokenType) {
255 void UnexpectedToken(object value, JsonTokenType tokenType) {
256 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
256 throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
257 }
257 }
258
258
259
259
260 /// <summary>
260 /// <summary>
261 /// ΠŸΡ€ΠΈΠ·Π½Π°ΠΊ ΠΊΠΎΠ½Ρ†Π° ΠΏΠΎΡ‚ΠΎΠΊΠ°
261 /// ΠŸΡ€ΠΈΠ·Π½Π°ΠΊ ΠΊΠΎΠ½Ρ†Π° ΠΏΠΎΡ‚ΠΎΠΊΠ°
262 /// </summary>
262 /// </summary>
263 public bool Eof {
263 public bool Eof {
264 get;
264 get;
265 private set;
265 private set;
266 }
266 }
267
267
268 protected override void Dispose(bool disposing) {
268 protected override void Dispose(bool disposing) {
269 if (disposing)
269 if (disposing)
270 m_scanner.Dispose();
270 m_scanner.Dispose();
271 }
271 }
272
272
273 /// <summary>
273 /// <summary>
274 /// ΠŸΠ΅Ρ€Π΅Ρ…ΠΎΠ΄ΠΈΡ‚ Π² ΠΊΠΎΠ½Π΅Ρ† Ρ‚Π΅ΠΊΡƒΡ‰Π΅Π³ΠΎ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Π°.
274 /// ΠŸΠ΅Ρ€Π΅Ρ…ΠΎΠ΄ΠΈΡ‚ Π² ΠΊΠΎΠ½Π΅Ρ† Ρ‚Π΅ΠΊΡƒΡ‰Π΅Π³ΠΎ ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Π°.
275 /// </summary>
275 /// </summary>
276 public void SeekElementEnd() {
276 public void SeekElementEnd() {
277 var level = Level - 1;
277 var level = Level - 1;
278
278
279 Debug.Assert(level >= 0);
279 Debug.Assert(level >= 0);
280
280
281 while (Level != level)
281 while (Level != level)
282 Read();
282 Read();
283 }
283 }
284
284
285 public static JsonReader Create(string file, Encoding encoding) {
285 public static JsonReader Create(string file, Encoding encoding) {
286 return new JsonReader(JsonTextScanner.Create(file, encoding));
286 return new JsonReader(JsonTextScanner.Create(file, encoding));
287 }
287 }
288
288
289 public static JsonReader Create(string file) {
289 public static JsonReader Create(string file) {
290 return new JsonReader(JsonTextScanner.Create(file));
290 return new JsonReader(JsonTextScanner.Create(file));
291 }
291 }
292
292
293 public static JsonReader Create(Stream stream, Encoding encoding) {
293 public static JsonReader Create(Stream stream, Encoding encoding) {
294 return new JsonReader(JsonTextScanner.Create(stream, encoding));
294 return new JsonReader(JsonTextScanner.Create(stream, encoding));
295 }
295 }
296
296
297 public static JsonReader Create(Stream stream) {
297 public static JsonReader Create(Stream stream) {
298 return new JsonReader(JsonTextScanner.Create(stream));
298 return new JsonReader(JsonTextScanner.Create(stream));
299 }
299 }
300
300
301 public static JsonReader Create(TextReader reader) {
301 public static JsonReader Create(TextReader reader) {
302 return new JsonReader(JsonTextScanner.Create(reader));
302 return new JsonReader(JsonTextScanner.Create(reader));
303 }
303 }
304
304
305 public static JsonReader ParseString(string data) {
305 public static JsonReader ParseString(string data) {
306 return new JsonReader(JsonStringScanner.Create(data));
306 return new JsonReader(JsonStringScanner.Create(data));
307 }
307 }
308
308
309 public static JsonReader ParseString(string data, int offset, int length) {
309 public static JsonReader ParseString(string data, int offset, int length) {
310 return new JsonReader(JsonStringScanner.Create(data, offset, length));
310 return new JsonReader(JsonStringScanner.Create(data, offset, length));
311 }
311 }
312
312
313 public static JsonReader ParseString(char[] data, int offset, int lenght) {
313 public static JsonReader ParseString(char[] data, int offset, int lenght) {
314 return new JsonReader(JsonStringScanner.Create(data, offset, lenght));
314 return new JsonReader(JsonStringScanner.Create(data, offset, lenght));
315 }
315 }
316 }
316 }
317
317
318 }
318 }
@@ -1,190 +1,190
1 using System;
1 using System;
2 using System.Globalization;
2 using System.Globalization;
3 using Implab.Automaton;
3 using Implab.Automaton;
4 using System.Text;
4 using System.Text;
5 using Implab.Components;
5 using Implab.Components;
6 using System.IO;
6 using System.IO;
7
7
8 namespace Implab.Formats.Json {
8 namespace Implab.Formats.Json {
9 /// <summary>
9 /// <summary>
10 /// Π‘ΠΊΠ°Π½Π½Π΅Ρ€ (лСксСр), Ρ€Π°Π·Π±ΠΈΠ²Π°ΡŽΡ‰ΠΈΠΉ ΠΏΠΎΡ‚ΠΎΠΊ символов Π½Π° Ρ‚ΠΎΠΊΠ΅Π½Ρ‹ JSON.
10 /// Π‘ΠΊΠ°Π½Π½Π΅Ρ€ (лСксСр), Ρ€Π°Π·Π±ΠΈΠ²Π°ΡŽΡ‰ΠΈΠΉ ΠΏΠΎΡ‚ΠΎΠΊ символов Π½Π° Ρ‚ΠΎΠΊΠ΅Π½Ρ‹ JSON.
11 /// </summary>
11 /// </summary>
12 public abstract class JsonScanner : Disposable {
12 public abstract class JsonScanner : Disposable {
13 readonly InputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
13 readonly FastInputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
14 readonly InputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
14 readonly FastInputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
15
15
16 readonly char[] m_unescapeBuf = new char[4];
16 readonly char[] m_unescapeBuf = new char[4];
17 readonly char[] m_buffer;
17 readonly char[] m_buffer;
18 int m_length;
18 int m_length;
19 int m_pos;
19 int m_pos;
20 readonly StringBuilder m_tokenBuilder = new StringBuilder();
20 readonly StringBuilder m_tokenBuilder = new StringBuilder();
21
21
22 protected JsonScanner(char[] buffer, int pos, int length) {
22 protected JsonScanner(char[] buffer, int pos, int length) {
23 m_buffer = buffer;
23 m_buffer = buffer;
24 m_pos = pos;
24 m_pos = pos;
25 m_length = length;
25 m_length = length;
26 }
26 }
27
27
28 bool ReadChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
28 bool ReadChunk(FastInputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
29 scanner.ResetState();
29 scanner.ResetState();
30
30
31 while(scanner.Scan(m_buffer, m_pos, m_length)) {
31 while(scanner.Scan(m_buffer, m_pos, m_length)) {
32 // scanner requests new data
32 // scanner requests new data
33
33
34 if (m_pos != m_length) // capture results for the future
34 if (m_pos != m_length) // capture results for the future
35 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
35 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
36
36
37 // read next data
37 // read next data
38 m_length = Read(m_buffer, 0, m_buffer.Length);
38 m_length = Read(m_buffer, 0, m_buffer.Length);
39
39
40 if (m_length == 0) {
40 if (m_length == 0) {
41 // no data is read
41 // no data is read
42 if (scanner.Position == m_pos) {
42 if (scanner.Position == m_pos) {
43 // scanned hasn't moved, that's the end
43 // scanned hasn't moved, that's the end
44 m_pos = 0;
44 m_pos = 0;
45 tokenType = JsonGrammar.TokenType.None;
45 tokenType = JsonGrammar.TokenType.None;
46 return false;
46 return false;
47 }
47 }
48
48
49 if (scanner.IsFinal) {
49 if (scanner.IsFinal) {
50 m_pos = 0;
50 m_pos = 0;
51 tokenType = scanner.Tag;
51 tokenType = scanner.Tag;
52 return true;
52 return true;
53 } else {
53 } else {
54 throw new ParserException("Unexpected EOF");
54 throw new ParserException("Unexpected EOF");
55 }
55 }
56 }
56 }
57
57
58 m_pos = 0;
58 m_pos = 0;
59 }
59 }
60 var scannerPos = scanner.Position;
60 var scannerPos = scanner.Position;
61
61
62 // scanner stops as scannerPos
62 // scanner stops as scannerPos
63 if (!scanner.IsFinal)
63 if (!scanner.IsFinal)
64 throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
64 throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
65
65
66 tokenType = scanner.Tag;
66 tokenType = scanner.Tag;
67 if (scannerPos != m_pos && tokenType == JsonGrammar.TokenType.Number || tokenType == JsonGrammar.TokenType.Literal)
67 if (scannerPos != m_pos && tokenType == JsonGrammar.TokenType.Number || tokenType == JsonGrammar.TokenType.Literal)
68 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
68 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
69
69
70 m_pos = scannerPos;
70 m_pos = scannerPos;
71 return true;
71 return true;
72 }
72 }
73
73
74 bool ReadStringChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
74 bool ReadStringChunk(FastInputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
75 scanner.ResetState();
75 scanner.ResetState();
76
76
77 while (scanner.Scan(m_buffer, m_pos, m_length)) {
77 while (scanner.Scan(m_buffer, m_pos, m_length)) {
78 // scanner requests new data
78 // scanner requests new data
79
79
80 if (m_pos != m_length) // capture results for the future
80 if (m_pos != m_length) // capture results for the future
81 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
81 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
82
82
83 // read next data
83 // read next data
84 m_length = Read(m_buffer, 0, m_buffer.Length);
84 m_length = Read(m_buffer, 0, m_buffer.Length);
85
85
86 if (m_length == 0) {
86 if (m_length == 0) {
87 // no data is read
87 // no data is read
88 if (scanner.Position == m_pos) {
88 if (scanner.Position == m_pos) {
89 // scanned hasn't moved, that's the end
89 // scanned hasn't moved, that's the end
90 m_pos = 0;
90 m_pos = 0;
91 tokenType = JsonGrammar.TokenType.None;
91 tokenType = JsonGrammar.TokenType.None;
92 return false;
92 return false;
93 }
93 }
94
94
95 if (scanner.IsFinal) {
95 if (scanner.IsFinal) {
96 m_pos = 0;
96 m_pos = 0;
97 tokenType = scanner.Tag;
97 tokenType = scanner.Tag;
98 return true;
98 return true;
99 } else {
99 } else {
100 throw new ParserException("Unexpected EOF");
100 throw new ParserException("Unexpected EOF");
101 }
101 }
102 }
102 }
103
103
104 m_pos = 0;
104 m_pos = 0;
105 }
105 }
106 var scannerPos = scanner.Position;
106 var scannerPos = scanner.Position;
107
107
108 // scanner stops as scannerPos
108 // scanner stops as scannerPos
109 if (!scanner.IsFinal)
109 if (!scanner.IsFinal)
110 throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
110 throw new ParserException($"Unexpected character '{m_buffer[scannerPos]}'");
111
111
112 if (scannerPos != m_pos) {
112 if (scannerPos != m_pos) {
113 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
113 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
114 m_pos = scannerPos;
114 m_pos = scannerPos;
115 }
115 }
116 tokenType = scanner.Tag;
116 tokenType = scanner.Tag;
117 return true;
117 return true;
118 }
118 }
119
119
120 protected abstract int Read(char[] buffer, int offset, int size);
120 protected abstract int Read(char[] buffer, int offset, int size);
121
121
122
122
123 /// <summary>
123 /// <summary>
124 /// Π§ΠΈΡ‚Π°Π΅Ρ‚ ΡΠ»Π΅Π΄ΡƒΡŽΡ‰ΠΈΠΉ лСксичСский элСмСнт ΠΈΠ· Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ….
124 /// Π§ΠΈΡ‚Π°Π΅Ρ‚ ΡΠ»Π΅Π΄ΡƒΡŽΡ‰ΠΈΠΉ лСксичСский элСмСнт ΠΈΠ· Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ….
125 /// </summary>
125 /// </summary>
126 /// <param name="tokenValue">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Π·Π½Π°Ρ‡Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
126 /// <param name="tokenValue">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Π·Π½Π°Ρ‡Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
127 /// <param name="tokenType">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Ρ‚ΠΈΠΏ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
127 /// <param name="tokenType">Π’ΠΎΠ·Π²Ρ€Π°Ρ‰Π°Π΅Ρ‚ Ρ‚ΠΈΠΏ ΠΏΡ€ΠΎΡ‡ΠΈΡ‚Π°Π½Π½ΠΎΠ³ΠΎ Ρ‚ΠΎΠΊΠ΅Π½Π°.</param>
128 /// <returns><c>true</c> - Ρ‡Ρ‚Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΠΈΠ·Π²Π΅Π΄Π΅Π½ΠΎ ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ. <c>false</c> - достигнут ΠΊΠΎΠ½Π΅Ρ† Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ…</returns>
128 /// <returns><c>true</c> - Ρ‡Ρ‚Π΅Π½ΠΈΠ΅ ΠΏΡ€ΠΎΠΈΠ·Π²Π΅Π΄Π΅Π½ΠΎ ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ. <c>false</c> - достигнут ΠΊΠΎΠ½Π΅Ρ† Π²Ρ…ΠΎΠ΄Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ…</returns>
129 /// <remarks>Π’ случС Ссли Ρ‚ΠΎΠΊΠ΅Π½ Π½Π΅ распознаСтся, Π²ΠΎΠ·Π½ΠΈΠΊΠ°Π΅Ρ‚ ΠΈΡΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅. ЗначСния Ρ‚ΠΎΠΊΠ΅Π½ΠΎΠ² ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ, Ρ‚.Π΅.
129 /// <remarks>Π’ случС Ссли Ρ‚ΠΎΠΊΠ΅Π½ Π½Π΅ распознаСтся, Π²ΠΎΠ·Π½ΠΈΠΊΠ°Π΅Ρ‚ ΠΈΡΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅. ЗначСния Ρ‚ΠΎΠΊΠ΅Π½ΠΎΠ² ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ, Ρ‚.Π΅.
130 /// Π² строках ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ экранированныС символы, числа становтся Ρ‚ΠΈΠΏΠ° double.</remarks>
130 /// Π² строках ΠΎΠ±Ρ€Π°Π±Π°Ρ‚Ρ‹Π²Π°ΡŽΡ‚ΡΡ экранированныС символы, числа становтся Ρ‚ΠΈΠΏΠ° double.</remarks>
131 public bool ReadToken(out string tokenValue, out JsonTokenType tokenType) {
131 public bool ReadToken(out string tokenValue, out JsonTokenType tokenType) {
132 JsonGrammar.TokenType tag;
132 JsonGrammar.TokenType tag;
133 m_tokenBuilder.Clear();
133 m_tokenBuilder.Clear();
134 while (ReadChunk(m_jsonContext, out tag)) {
134 while (ReadChunk(m_jsonContext, out tag)) {
135 switch (tag) {
135 switch (tag) {
136 case JsonGrammar.TokenType.StringBound:
136 case JsonGrammar.TokenType.StringBound:
137 tokenValue = ReadString();
137 tokenValue = ReadString();
138 tokenType = JsonTokenType.String;
138 tokenType = JsonTokenType.String;
139 break;
139 break;
140 case JsonGrammar.TokenType.Number:
140 case JsonGrammar.TokenType.Number:
141 tokenValue = m_tokenBuilder.ToString();
141 tokenValue = m_tokenBuilder.ToString();
142 tokenType = JsonTokenType.Number;
142 tokenType = JsonTokenType.Number;
143 break;
143 break;
144 case JsonGrammar.TokenType.Literal:
144 case JsonGrammar.TokenType.Literal:
145 tokenType = JsonTokenType.Literal;
145 tokenType = JsonTokenType.Literal;
146 tokenValue = m_tokenBuilder.ToString();
146 tokenValue = m_tokenBuilder.ToString();
147 break;
147 break;
148 case JsonGrammar.TokenType.Whitespace:
148 case JsonGrammar.TokenType.Whitespace:
149 m_tokenBuilder.Clear();
149 m_tokenBuilder.Clear();
150 continue;
150 continue;
151 default:
151 default:
152 tokenType = (JsonTokenType)tag;
152 tokenType = (JsonTokenType)tag;
153 tokenValue = null;
153 tokenValue = null;
154 break;
154 break;
155 }
155 }
156 return true;
156 return true;
157 }
157 }
158 tokenValue = null;
158 tokenValue = null;
159 tokenType = JsonTokenType.None;
159 tokenType = JsonTokenType.None;
160 return false;
160 return false;
161 }
161 }
162
162
163 string ReadString() {
163 string ReadString() {
164 JsonGrammar.TokenType tag;
164 JsonGrammar.TokenType tag;
165 m_tokenBuilder.Clear();
165 m_tokenBuilder.Clear();
166
166
167 while (ReadStringChunk(m_stringContext, out tag)) {
167 while (ReadStringChunk(m_stringContext, out tag)) {
168 switch (tag) {
168 switch (tag) {
169 case JsonGrammar.TokenType.StringBound:
169 case JsonGrammar.TokenType.StringBound:
170 m_tokenBuilder.Length--;
170 m_tokenBuilder.Length--;
171 return m_tokenBuilder.ToString();
171 return m_tokenBuilder.ToString();
172 case JsonGrammar.TokenType.UnescapedChar:
172 case JsonGrammar.TokenType.UnescapedChar:
173 break;
173 break;
174 case JsonGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
174 case JsonGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
175 m_tokenBuilder.CopyTo(m_tokenBuilder.Length - 4, m_unescapeBuf, 0, 4);
175 m_tokenBuilder.CopyTo(m_tokenBuilder.Length - 4, m_unescapeBuf, 0, 4);
176 m_tokenBuilder.Length -= 6;
176 m_tokenBuilder.Length -= 6;
177 m_tokenBuilder.Append(StringTranslator.TranslateHexUnicode(m_unescapeBuf, 0));
177 m_tokenBuilder.Append(StringTranslator.TranslateHexUnicode(m_unescapeBuf, 0));
178 break;
178 break;
179 case JsonGrammar.TokenType.EscapedChar: // \t - escape sequence
179 case JsonGrammar.TokenType.EscapedChar: // \t - escape sequence
180 var ch = m_tokenBuilder[m_tokenBuilder.Length-1];
180 var ch = m_tokenBuilder[m_tokenBuilder.Length-1];
181 m_tokenBuilder.Length -= 2;
181 m_tokenBuilder.Length -= 2;
182 m_tokenBuilder.Append(StringTranslator.TranslateEscapedChar(ch));
182 m_tokenBuilder.Append(StringTranslator.TranslateEscapedChar(ch));
183 break;
183 break;
184 }
184 }
185 }
185 }
186
186
187 throw new ParserException("Unexpected end of data");
187 throw new ParserException("Unexpected end of data");
188 }
188 }
189 }
189 }
190 }
190 }
@@ -1,188 +1,189
1 ο»Ώ<?xml version="1.0" encoding="utf-8"?>
1 ο»Ώ<?xml version="1.0" encoding="utf-8"?>
2 <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2 <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 <PropertyGroup>
3 <PropertyGroup>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
6 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
7 <OutputType>Library</OutputType>
7 <OutputType>Library</OutputType>
8 <RootNamespace>Implab</RootNamespace>
8 <RootNamespace>Implab</RootNamespace>
9 <AssemblyName>Implab</AssemblyName>
9 <AssemblyName>Implab</AssemblyName>
10 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
10 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
11 <TargetFrameworkProfile />
11 <TargetFrameworkProfile />
12 </PropertyGroup>
12 </PropertyGroup>
13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
13 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
14 <DebugSymbols>true</DebugSymbols>
14 <DebugSymbols>true</DebugSymbols>
15 <DebugType>full</DebugType>
15 <DebugType>full</DebugType>
16 <Optimize>false</Optimize>
16 <Optimize>false</Optimize>
17 <OutputPath>bin\Debug</OutputPath>
17 <OutputPath>bin\Debug</OutputPath>
18 <DefineConstants>TRACE;DEBUG;NET_4_5</DefineConstants>
18 <DefineConstants>TRACE;DEBUG;NET_4_5</DefineConstants>
19 <ErrorReport>prompt</ErrorReport>
19 <ErrorReport>prompt</ErrorReport>
20 <WarningLevel>4</WarningLevel>
20 <WarningLevel>4</WarningLevel>
21 <ConsolePause>false</ConsolePause>
21 <ConsolePause>false</ConsolePause>
22 <RunCodeAnalysis>true</RunCodeAnalysis>
22 <RunCodeAnalysis>true</RunCodeAnalysis>
23 </PropertyGroup>
23 </PropertyGroup>
24 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
24 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
25 <DebugType>full</DebugType>
25 <DebugType>full</DebugType>
26 <Optimize>true</Optimize>
26 <Optimize>true</Optimize>
27 <OutputPath>bin\Release</OutputPath>
27 <OutputPath>bin\Release</OutputPath>
28 <DefineConstants>NET_4_5</DefineConstants>
28 <DefineConstants>NET_4_5</DefineConstants>
29 <ErrorReport>prompt</ErrorReport>
29 <ErrorReport>prompt</ErrorReport>
30 <WarningLevel>4</WarningLevel>
30 <WarningLevel>4</WarningLevel>
31 <ConsolePause>false</ConsolePause>
31 <ConsolePause>false</ConsolePause>
32 </PropertyGroup>
32 </PropertyGroup>
33 <PropertyGroup>
33 <PropertyGroup>
34 <SignAssembly>true</SignAssembly>
34 <SignAssembly>true</SignAssembly>
35 </PropertyGroup>
35 </PropertyGroup>
36 <PropertyGroup>
36 <PropertyGroup>
37 <AssemblyOriginatorKeyFile>implab.snk</AssemblyOriginatorKeyFile>
37 <AssemblyOriginatorKeyFile>implab.snk</AssemblyOriginatorKeyFile>
38 </PropertyGroup>
38 </PropertyGroup>
39 <ItemGroup>
39 <ItemGroup>
40 <Reference Include="System" />
40 <Reference Include="System" />
41 <Reference Include="System.Xml" />
41 <Reference Include="System.Xml" />
42 <Reference Include="mscorlib" />
42 <Reference Include="mscorlib" />
43 <Reference Include="System.Xml.Linq" />
43 <Reference Include="System.Xml.Linq" />
44 </ItemGroup>
44 </ItemGroup>
45 <ItemGroup>
45 <ItemGroup>
46 <Compile Include="Components\StateChangeEventArgs.cs" />
46 <Compile Include="Components\StateChangeEventArgs.cs" />
47 <Compile Include="CustomEqualityComparer.cs" />
47 <Compile Include="CustomEqualityComparer.cs" />
48 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
48 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
49 <Compile Include="Diagnostics\LogChannel.cs" />
49 <Compile Include="Diagnostics\LogChannel.cs" />
50 <Compile Include="Diagnostics\LogicalOperation.cs" />
50 <Compile Include="Diagnostics\LogicalOperation.cs" />
51 <Compile Include="Diagnostics\TextFileListener.cs" />
51 <Compile Include="Diagnostics\TextFileListener.cs" />
52 <Compile Include="Diagnostics\Trace.cs" />
52 <Compile Include="Diagnostics\Trace.cs" />
53 <Compile Include="Diagnostics\TraceLog.cs" />
53 <Compile Include="Diagnostics\TraceLog.cs" />
54 <Compile Include="Diagnostics\TraceEvent.cs" />
54 <Compile Include="Diagnostics\TraceEvent.cs" />
55 <Compile Include="Diagnostics\TraceEventType.cs" />
55 <Compile Include="Diagnostics\TraceEventType.cs" />
56 <Compile Include="Diagnostics\TraceSourceAttribute.cs" />
56 <Compile Include="Diagnostics\TraceSourceAttribute.cs" />
57 <Compile Include="Formats\CharMap.cs" />
57 <Compile Include="Formats\CharMap.cs" />
58 <Compile Include="Formats\FastInpurScanner.cs" />
58 <Compile Include="Formats\InputScanner.cs" />
59 <Compile Include="Formats\InputScanner.cs" />
59 <Compile Include="Formats\Json\JsonStringScanner.cs" />
60 <Compile Include="Formats\Json\JsonStringScanner.cs" />
60 <Compile Include="Formats\Json\JsonTextScanner.cs" />
61 <Compile Include="Formats\Json\JsonTextScanner.cs" />
61 <Compile Include="ICancellable.cs" />
62 <Compile Include="ICancellable.cs" />
62 <Compile Include="IProgressHandler.cs" />
63 <Compile Include="IProgressHandler.cs" />
63 <Compile Include="IProgressNotifier.cs" />
64 <Compile Include="IProgressNotifier.cs" />
64 <Compile Include="IPromiseT.cs" />
65 <Compile Include="IPromiseT.cs" />
65 <Compile Include="IPromise.cs" />
66 <Compile Include="IPromise.cs" />
66 <Compile Include="IServiceLocator.cs" />
67 <Compile Include="IServiceLocator.cs" />
67 <Compile Include="ITaskController.cs" />
68 <Compile Include="ITaskController.cs" />
68 <Compile Include="Parallels\DispatchPool.cs" />
69 <Compile Include="Parallels\DispatchPool.cs" />
69 <Compile Include="Parallels\ArrayTraits.cs" />
70 <Compile Include="Parallels\ArrayTraits.cs" />
70 <Compile Include="Parallels\SimpleAsyncQueue.cs" />
71 <Compile Include="Parallels\SimpleAsyncQueue.cs" />
71 <Compile Include="Parallels\WorkerPool.cs" />
72 <Compile Include="Parallels\WorkerPool.cs" />
72 <Compile Include="ProgressInitEventArgs.cs" />
73 <Compile Include="ProgressInitEventArgs.cs" />
73 <Compile Include="Properties\AssemblyInfo.cs" />
74 <Compile Include="Properties\AssemblyInfo.cs" />
74 <Compile Include="Parallels\AsyncPool.cs" />
75 <Compile Include="Parallels\AsyncPool.cs" />
75 <Compile Include="Safe.cs" />
76 <Compile Include="Safe.cs" />
76 <Compile Include="SyncContextPromise.cs" />
77 <Compile Include="SyncContextPromise.cs" />
77 <Compile Include="ValueEventArgs.cs" />
78 <Compile Include="ValueEventArgs.cs" />
78 <Compile Include="PromiseExtensions.cs" />
79 <Compile Include="PromiseExtensions.cs" />
79 <Compile Include="SyncContextPromiseT.cs" />
80 <Compile Include="SyncContextPromiseT.cs" />
80 <Compile Include="Diagnostics\OperationContext.cs" />
81 <Compile Include="Diagnostics\OperationContext.cs" />
81 <Compile Include="Diagnostics\TraceContext.cs" />
82 <Compile Include="Diagnostics\TraceContext.cs" />
82 <Compile Include="Diagnostics\LogEventArgs.cs" />
83 <Compile Include="Diagnostics\LogEventArgs.cs" />
83 <Compile Include="Diagnostics\LogEventArgsT.cs" />
84 <Compile Include="Diagnostics\LogEventArgsT.cs" />
84 <Compile Include="Diagnostics\Extensions.cs" />
85 <Compile Include="Diagnostics\Extensions.cs" />
85 <Compile Include="PromiseEventType.cs" />
86 <Compile Include="PromiseEventType.cs" />
86 <Compile Include="Parallels\AsyncQueue.cs" />
87 <Compile Include="Parallels\AsyncQueue.cs" />
87 <Compile Include="PromiseT.cs" />
88 <Compile Include="PromiseT.cs" />
88 <Compile Include="IDeferred.cs" />
89 <Compile Include="IDeferred.cs" />
89 <Compile Include="IDeferredT.cs" />
90 <Compile Include="IDeferredT.cs" />
90 <Compile Include="Promise.cs" />
91 <Compile Include="Promise.cs" />
91 <Compile Include="PromiseTransientException.cs" />
92 <Compile Include="PromiseTransientException.cs" />
92 <Compile Include="Parallels\Signal.cs" />
93 <Compile Include="Parallels\Signal.cs" />
93 <Compile Include="Parallels\SharedLock.cs" />
94 <Compile Include="Parallels\SharedLock.cs" />
94 <Compile Include="Diagnostics\ILogWriter.cs" />
95 <Compile Include="Diagnostics\ILogWriter.cs" />
95 <Compile Include="Diagnostics\ListenerBase.cs" />
96 <Compile Include="Diagnostics\ListenerBase.cs" />
96 <Compile Include="Parallels\BlockingQueue.cs" />
97 <Compile Include="Parallels\BlockingQueue.cs" />
97 <Compile Include="AbstractEvent.cs" />
98 <Compile Include="AbstractEvent.cs" />
98 <Compile Include="AbstractPromise.cs" />
99 <Compile Include="AbstractPromise.cs" />
99 <Compile Include="AbstractPromiseT.cs" />
100 <Compile Include="AbstractPromiseT.cs" />
100 <Compile Include="FuncTask.cs" />
101 <Compile Include="FuncTask.cs" />
101 <Compile Include="FuncTaskBase.cs" />
102 <Compile Include="FuncTaskBase.cs" />
102 <Compile Include="FuncTaskT.cs" />
103 <Compile Include="FuncTaskT.cs" />
103 <Compile Include="ActionChainTaskBase.cs" />
104 <Compile Include="ActionChainTaskBase.cs" />
104 <Compile Include="ActionChainTask.cs" />
105 <Compile Include="ActionChainTask.cs" />
105 <Compile Include="ActionChainTaskT.cs" />
106 <Compile Include="ActionChainTaskT.cs" />
106 <Compile Include="FuncChainTaskBase.cs" />
107 <Compile Include="FuncChainTaskBase.cs" />
107 <Compile Include="FuncChainTask.cs" />
108 <Compile Include="FuncChainTask.cs" />
108 <Compile Include="FuncChainTaskT.cs" />
109 <Compile Include="FuncChainTaskT.cs" />
109 <Compile Include="ActionTaskBase.cs" />
110 <Compile Include="ActionTaskBase.cs" />
110 <Compile Include="ActionTask.cs" />
111 <Compile Include="ActionTask.cs" />
111 <Compile Include="ActionTaskT.cs" />
112 <Compile Include="ActionTaskT.cs" />
112 <Compile Include="ICancellationToken.cs" />
113 <Compile Include="ICancellationToken.cs" />
113 <Compile Include="SuccessPromise.cs" />
114 <Compile Include="SuccessPromise.cs" />
114 <Compile Include="SuccessPromiseT.cs" />
115 <Compile Include="SuccessPromiseT.cs" />
115 <Compile Include="PromiseAwaiterT.cs" />
116 <Compile Include="PromiseAwaiterT.cs" />
116 <Compile Include="PromiseAwaiter.cs" />
117 <Compile Include="PromiseAwaiter.cs" />
117 <Compile Include="Components\ComponentContainer.cs" />
118 <Compile Include="Components\ComponentContainer.cs" />
118 <Compile Include="Components\Disposable.cs" />
119 <Compile Include="Components\Disposable.cs" />
119 <Compile Include="Components\DisposablePool.cs" />
120 <Compile Include="Components\DisposablePool.cs" />
120 <Compile Include="Components\ObjectPool.cs" />
121 <Compile Include="Components\ObjectPool.cs" />
121 <Compile Include="Components\ServiceLocator.cs" />
122 <Compile Include="Components\ServiceLocator.cs" />
122 <Compile Include="Components\IInitializable.cs" />
123 <Compile Include="Components\IInitializable.cs" />
123 <Compile Include="TaskController.cs" />
124 <Compile Include="TaskController.cs" />
124 <Compile Include="Components\App.cs" />
125 <Compile Include="Components\App.cs" />
125 <Compile Include="Components\IRunnable.cs" />
126 <Compile Include="Components\IRunnable.cs" />
126 <Compile Include="Components\ExecutionState.cs" />
127 <Compile Include="Components\ExecutionState.cs" />
127 <Compile Include="Components\RunnableComponent.cs" />
128 <Compile Include="Components\RunnableComponent.cs" />
128 <Compile Include="Components\IFactory.cs" />
129 <Compile Include="Components\IFactory.cs" />
129 <Compile Include="Automaton\IAlphabet.cs" />
130 <Compile Include="Automaton\IAlphabet.cs" />
130 <Compile Include="Automaton\ParserException.cs" />
131 <Compile Include="Automaton\ParserException.cs" />
131 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
132 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
132 <Compile Include="Automaton\IAlphabetBuilder.cs" />
133 <Compile Include="Automaton\IAlphabetBuilder.cs" />
133 <Compile Include="Automaton\RegularExpressions\AltToken.cs" />
134 <Compile Include="Automaton\RegularExpressions\AltToken.cs" />
134 <Compile Include="Automaton\RegularExpressions\BinaryToken.cs" />
135 <Compile Include="Automaton\RegularExpressions\BinaryToken.cs" />
135 <Compile Include="Automaton\RegularExpressions\CatToken.cs" />
136 <Compile Include="Automaton\RegularExpressions\CatToken.cs" />
136 <Compile Include="Automaton\RegularExpressions\StarToken.cs" />
137 <Compile Include="Automaton\RegularExpressions\StarToken.cs" />
137 <Compile Include="Automaton\RegularExpressions\SymbolToken.cs" />
138 <Compile Include="Automaton\RegularExpressions\SymbolToken.cs" />
138 <Compile Include="Automaton\RegularExpressions\EmptyToken.cs" />
139 <Compile Include="Automaton\RegularExpressions\EmptyToken.cs" />
139 <Compile Include="Automaton\RegularExpressions\Token.cs" />
140 <Compile Include="Automaton\RegularExpressions\Token.cs" />
140 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
141 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
141 <Compile Include="Automaton\AutomatonTransition.cs" />
142 <Compile Include="Automaton\AutomatonTransition.cs" />
142 <Compile Include="Formats\Json\JsonElementContext.cs" />
143 <Compile Include="Formats\Json\JsonElementContext.cs" />
143 <Compile Include="Formats\Json\JsonElementType.cs" />
144 <Compile Include="Formats\Json\JsonElementType.cs" />
144 <Compile Include="Formats\Json\JsonGrammar.cs" />
145 <Compile Include="Formats\Json\JsonGrammar.cs" />
145 <Compile Include="Formats\Json\JsonReader.cs" />
146 <Compile Include="Formats\Json\JsonReader.cs" />
146 <Compile Include="Formats\Json\JsonScanner.cs" />
147 <Compile Include="Formats\Json\JsonScanner.cs" />
147 <Compile Include="Formats\Json\JsonTokenType.cs" />
148 <Compile Include="Formats\Json\JsonTokenType.cs" />
148 <Compile Include="Formats\Json\JsonWriter.cs" />
149 <Compile Include="Formats\Json\JsonWriter.cs" />
149 <Compile Include="Formats\Json\StringTranslator.cs" />
150 <Compile Include="Formats\Json\StringTranslator.cs" />
150 <Compile Include="Automaton\MapAlphabet.cs" />
151 <Compile Include="Automaton\MapAlphabet.cs" />
151 <Compile Include="Formats\CharAlphabet.cs" />
152 <Compile Include="Formats\CharAlphabet.cs" />
152 <Compile Include="Formats\ByteAlphabet.cs" />
153 <Compile Include="Formats\ByteAlphabet.cs" />
153 <Compile Include="Automaton\IDFATable.cs" />
154 <Compile Include="Automaton\IDFATable.cs" />
154 <Compile Include="Automaton\IDFATableBuilder.cs" />
155 <Compile Include="Automaton\IDFATableBuilder.cs" />
155 <Compile Include="Automaton\DFATable.cs" />
156 <Compile Include="Automaton\DFATable.cs" />
156 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
157 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
157 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
158 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
158 <Compile Include="Formats\Grammar.cs" />
159 <Compile Include="Formats\Grammar.cs" />
159 <Compile Include="Automaton\RegularExpressions\EndTokenT.cs" />
160 <Compile Include="Automaton\RegularExpressions\EndTokenT.cs" />
160 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
161 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
161 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitorT.cs" />
162 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitorT.cs" />
162 <Compile Include="Automaton\AutomatonConst.cs" />
163 <Compile Include="Automaton\AutomatonConst.cs" />
163 <Compile Include="Automaton\RegularExpressions\RegularDFA.cs" />
164 <Compile Include="Automaton\RegularExpressions\RegularDFA.cs" />
164 <Compile Include="Components\LazyAndWeak.cs" />
165 <Compile Include="Components\LazyAndWeak.cs" />
165 <Compile Include="AbstractTask.cs" />
166 <Compile Include="AbstractTask.cs" />
166 <Compile Include="AbstractTaskT.cs" />
167 <Compile Include="AbstractTaskT.cs" />
167 <Compile Include="FailedPromise.cs" />
168 <Compile Include="FailedPromise.cs" />
168 <Compile Include="FailedPromiseT.cs" />
169 <Compile Include="FailedPromiseT.cs" />
169 <Compile Include="Components\PollingComponent.cs" />
170 <Compile Include="Components\PollingComponent.cs" />
170 <Compile Include="Xml\JsonXmlReader.cs" />
171 <Compile Include="Xml\JsonXmlReader.cs" />
171 <Compile Include="Xml\JsonXmlReaderOptions.cs" />
172 <Compile Include="Xml\JsonXmlReaderOptions.cs" />
172 <Compile Include="Xml\JsonXmlReaderPosition.cs" />
173 <Compile Include="Xml\JsonXmlReaderPosition.cs" />
173 <Compile Include="Xml\SerializationHelpers.cs" />
174 <Compile Include="Xml\SerializationHelpers.cs" />
174 <Compile Include="Xml\SerializersPool.cs" />
175 <Compile Include="Xml\SerializersPool.cs" />
175 <Compile Include="Xml\XmlSimpleAttribute.cs" />
176 <Compile Include="Xml\XmlSimpleAttribute.cs" />
176 <Compile Include="Xml\XmlNameContext.cs" />
177 <Compile Include="Xml\XmlNameContext.cs" />
177 </ItemGroup>
178 </ItemGroup>
178 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
179 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
179 <ItemGroup>
180 <ItemGroup>
180 <None Include="Implab.nuspec">
181 <None Include="Implab.nuspec">
181 <SubType>Designer</SubType>
182 <SubType>Designer</SubType>
182 </None>
183 </None>
183 <None Include="implab.snk" />
184 <None Include="implab.snk" />
184 </ItemGroup>
185 </ItemGroup>
185 <ItemGroup>
186 <ItemGroup>
186 <Content Include="license.txt" />
187 <Content Include="license.txt" />
187 </ItemGroup>
188 </ItemGroup>
188 </Project> No newline at end of file
189 </Project>
@@ -1,626 +1,610
1 using Implab.Formats.Json;
1 using Implab.Formats.Json;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Globalization;
4 using System.Globalization;
5 using System.Linq;
5 using System.Linq;
6 using System.Xml;
6 using System.Xml;
7
7
8 namespace Implab.Xml {
8 namespace Implab.Xml {
9 public class JsonXmlReader : XmlReader {
9 public class JsonXmlReader : XmlReader {
10 struct JsonContext {
10 struct JsonContext {
11 public string localName;
11 public string localName;
12 public bool skip;
12 public bool skip;
13 }
13 }
14
14
15 JsonReader m_parser;
15 JsonReader m_parser;
16 JsonXmlReaderOptions m_options;
16 JsonXmlReaderOptions m_options;
17 JsonXmlReaderPosition m_position = JsonXmlReaderPosition.Initial;
17 JsonXmlReaderPosition m_position = JsonXmlReaderPosition.Initial;
18 XmlNameTable m_nameTable;
18 XmlNameTable m_nameTable;
19
19
20 readonly string m_jsonRootName;
20 readonly string m_jsonRootName;
21 readonly string m_jsonNamespace;
21 readonly string m_jsonNamespace;
22 readonly string m_jsonPrefix;
22 readonly string m_jsonPrefix;
23 readonly bool m_jsonFlattenArrays;
23 readonly bool m_jsonFlattenArrays;
24 readonly string m_jsonArrayItemName;
24 readonly string m_jsonArrayItemName;
25
25
26 string m_jsonLocalName;
26 string m_jsonLocalName;
27 string m_jsonValueName;
27 string m_jsonValueName;
28 bool m_jsonSkip; // indicates wheather to generate closing tag for objects or arrays
28 bool m_jsonSkip; // indicates wheather to generate closing tag for objects or arrays
29
29
30 readonly Stack<JsonContext> m_jsonNameStack = new Stack<JsonContext>();
30 readonly Stack<JsonContext> m_jsonNameStack = new Stack<JsonContext>();
31
31
32 XmlQualifiedName m_elementQName;
32 XmlQualifiedName m_elementQName;
33 string m_elementPrefix;
33 string m_elementPrefix;
34 int m_elementDepth;
34 int m_elementDepth;
35 bool m_elementIsEmpty;
35 bool m_elementIsEmpty;
36
36
37 XmlQualifiedName m_qName;
37 XmlQualifiedName m_qName;
38 string m_prefix;
38 string m_prefix;
39 int m_xmlDepth;
39 int m_xmlDepth;
40
40
41 XmlSimpleAttribute[] m_attributes;
41 XmlSimpleAttribute[] m_attributes;
42 object m_value;
42 string m_value;
43 bool m_isEmpty;
43 bool m_isEmpty;
44
44
45 XmlNodeType m_nodeType = XmlNodeType.None;
45 XmlNodeType m_nodeType = XmlNodeType.None;
46
46
47 bool m_isAttribute; // indicates that we are reading attribute nodes
47 bool m_isAttribute; // indicates that we are reading attribute nodes
48 int m_currentAttribute;
48 int m_currentAttribute;
49 bool m_currentAttributeRead;
49 bool m_currentAttributeRead;
50
50
51
51
52 XmlNameContext m_context;
52 XmlNameContext m_context;
53
53
54 readonly string m_xmlnsPrefix;
54 readonly string m_xmlnsPrefix;
55 readonly string m_xmlnsNamespace;
55 readonly string m_xmlnsNamespace;
56 readonly string m_xsiPrefix;
56 readonly string m_xsiPrefix;
57 readonly string m_xsiNamespace;
57 readonly string m_xsiNamespace;
58
58
59
59
60 public JsonXmlReader(JsonReader parser, JsonXmlReaderOptions options) {
60 public JsonXmlReader(JsonReader parser, JsonXmlReaderOptions options) {
61 Safe.ArgumentNotNull(parser, nameof(parser));
61 Safe.ArgumentNotNull(parser, nameof(parser));
62 m_parser = parser;
62 m_parser = parser;
63
63
64 m_options = options ?? new JsonXmlReaderOptions();
64 m_options = options ?? new JsonXmlReaderOptions();
65
65
66 m_jsonFlattenArrays = m_options.FlattenArrays;
66 m_jsonFlattenArrays = m_options.FlattenArrays;
67 m_nameTable = m_options.NameTable ?? new NameTable();
67 m_nameTable = m_options.NameTable ?? new NameTable();
68
68
69 m_jsonRootName = m_nameTable.Add(string.IsNullOrEmpty(m_options.RootName) ? "data" : m_options.RootName);
69 m_jsonRootName = m_nameTable.Add(string.IsNullOrEmpty(m_options.RootName) ? "data" : m_options.RootName);
70 m_jsonArrayItemName = m_nameTable.Add(string.IsNullOrEmpty(m_options.ArrayItemName) ? "item" : m_options.ArrayItemName);
70 m_jsonArrayItemName = m_nameTable.Add(string.IsNullOrEmpty(m_options.ArrayItemName) ? "item" : m_options.ArrayItemName);
71 m_jsonNamespace = m_nameTable.Add(m_options.NamespaceUri ?? string.Empty);
71 m_jsonNamespace = m_nameTable.Add(m_options.NamespaceUri ?? string.Empty);
72 m_jsonPrefix = m_nameTable.Add(m_options.NodesPrefix ?? string.Empty);
72 m_jsonPrefix = m_nameTable.Add(m_options.NodesPrefix ?? string.Empty);
73 m_xmlnsPrefix = m_nameTable.Add(XmlNameContext.XmlnsPrefix);
73 m_xmlnsPrefix = m_nameTable.Add(XmlNameContext.XmlnsPrefix);
74 m_xmlnsNamespace = m_nameTable.Add(XmlNameContext.XmlnsNamespace);
74 m_xmlnsNamespace = m_nameTable.Add(XmlNameContext.XmlnsNamespace);
75 m_xsiPrefix = m_nameTable.Add(XmlNameContext.XsiPrefix);
75 m_xsiPrefix = m_nameTable.Add(XmlNameContext.XsiPrefix);
76 m_xsiNamespace = m_nameTable.Add(XmlNameContext.XsiNamespace);
76 m_xsiNamespace = m_nameTable.Add(XmlNameContext.XsiNamespace);
77
77
78 // TODO validate m_jsonRootName, m_jsonArrayItemName
78 // TODO validate m_jsonRootName, m_jsonArrayItemName
79
79
80 m_context = new XmlNameContext(null, 0);
80 m_context = new XmlNameContext(null, 0);
81 }
81 }
82
82
83 public override int AttributeCount {
83 public override int AttributeCount {
84 get {
84 get {
85 return m_attributes == null ? 0 : m_attributes.Length;
85 return m_attributes == null ? 0 : m_attributes.Length;
86 }
86 }
87 }
87 }
88
88
89 public override string BaseURI {
89 public override string BaseURI {
90 get {
90 get {
91 return string.Empty;
91 return string.Empty;
92 }
92 }
93 }
93 }
94
94
95 public override int Depth {
95 public override int Depth {
96 get {
96 get {
97 return m_xmlDepth;
97 return m_xmlDepth;
98 }
98 }
99 }
99 }
100
100
101 public override bool EOF {
101 public override bool EOF {
102 get {
102 get {
103 return m_position == JsonXmlReaderPosition.Eof;
103 return m_position == JsonXmlReaderPosition.Eof;
104 }
104 }
105 }
105 }
106
106
107 public override bool IsEmptyElement {
107 public override bool IsEmptyElement {
108 get { return m_isEmpty; }
108 get { return m_isEmpty; }
109 }
109 }
110
110
111
111
112 public override string LocalName {
112 public override string LocalName {
113 get {
113 get {
114 return m_qName.Name;
114 return m_qName.Name;
115 }
115 }
116 }
116 }
117
117
118 public override string NamespaceURI {
118 public override string NamespaceURI {
119 get {
119 get {
120 return m_qName.Namespace;
120 return m_qName.Namespace;
121 }
121 }
122 }
122 }
123
123
124 public override XmlNameTable NameTable {
124 public override XmlNameTable NameTable {
125 get {
125 get {
126 return m_nameTable;
126 return m_nameTable;
127 }
127 }
128 }
128 }
129
129
130 public override XmlNodeType NodeType {
130 public override XmlNodeType NodeType {
131 get {
131 get {
132 return m_nodeType;
132 return m_nodeType;
133 }
133 }
134 }
134 }
135
135
136 public override string Prefix {
136 public override string Prefix {
137 get {
137 get {
138 return m_prefix;
138 return m_prefix;
139 }
139 }
140 }
140 }
141
141
142 public override ReadState ReadState {
142 public override ReadState ReadState {
143 get {
143 get {
144 switch (m_position) {
144 switch (m_position) {
145 case JsonXmlReaderPosition.Initial:
145 case JsonXmlReaderPosition.Initial:
146 return ReadState.Initial;
146 return ReadState.Initial;
147 case JsonXmlReaderPosition.Eof:
147 case JsonXmlReaderPosition.Eof:
148 return ReadState.EndOfFile;
148 return ReadState.EndOfFile;
149 case JsonXmlReaderPosition.Closed:
149 case JsonXmlReaderPosition.Closed:
150 return ReadState.Closed;
150 return ReadState.Closed;
151 case JsonXmlReaderPosition.Error:
151 case JsonXmlReaderPosition.Error:
152 return ReadState.Error;
152 return ReadState.Error;
153 default:
153 default:
154 return ReadState.Interactive;
154 return ReadState.Interactive;
155 };
155 };
156 }
156 }
157 }
157 }
158
158
159 public override string Value {
159 public override string Value {
160 get {
160 get {
161 return ConvertValueToString(m_value);
161 return m_value;
162 }
163 }
164
165 static string ConvertValueToString(object value) {
166 if (value == null)
167 return string.Empty;
168
169 switch (Convert.GetTypeCode(value)) {
170 case TypeCode.Double:
171 return ((double)value).ToString(CultureInfo.InvariantCulture);
172 case TypeCode.String:
173 return (string)value;
174 case TypeCode.Boolean:
175 return (bool)value ? "true" : "false";
176 default:
177 return value.ToString();
178 }
162 }
179 }
163 }
180
164
181 public override string GetAttribute(int i) {
165 public override string GetAttribute(int i) {
182 Safe.ArgumentInRange(i, 0, AttributeCount - 1, nameof(i));
166 Safe.ArgumentInRange(i, 0, AttributeCount - 1, nameof(i));
183 return ConvertValueToString(m_attributes[i].Value);
167 return m_attributes[i].Value;
184 }
168 }
185
169
186 public override string GetAttribute(string name) {
170 public override string GetAttribute(string name) {
187 if (m_attributes == null)
171 if (m_attributes == null)
188 return null;
172 return null;
189 var qName = m_context.Resolve(name);
173 var qName = m_context.Resolve(name);
190 var attr = Array.Find(m_attributes, x => x.QName == qName);
174 var attr = Array.Find(m_attributes, x => x.QName == qName);
191 var value = ConvertValueToString(attr?.Value);
175 var value = attr?.Value;
192 return value == string.Empty ? null : value;
176 return value == string.Empty ? null : value;
193 }
177 }
194
178
195 public override string GetAttribute(string name, string namespaceURI) {
179 public override string GetAttribute(string name, string namespaceURI) {
196 if (m_attributes == null)
180 if (m_attributes == null)
197 return null;
181 return null;
198 var qName = new XmlQualifiedName(name, namespaceURI);
182 var qName = new XmlQualifiedName(name, namespaceURI);
199 var attr = Array.Find(m_attributes, x => x.QName == qName);
183 var attr = Array.Find(m_attributes, x => x.QName == qName);
200 var value = ConvertValueToString(attr?.Value);
184 var value = attr?.Value;
201 return value == string.Empty ? null : value;
185 return value == string.Empty ? null : value;
202 }
186 }
203
187
204 public override string LookupNamespace(string prefix) {
188 public override string LookupNamespace(string prefix) {
205 return m_context.ResolvePrefix(prefix);
189 return m_context.ResolvePrefix(prefix);
206 }
190 }
207
191
208 public override bool MoveToAttribute(string name) {
192 public override bool MoveToAttribute(string name) {
209 if (m_attributes == null || m_attributes.Length == 0)
193 if (m_attributes == null || m_attributes.Length == 0)
210 return false;
194 return false;
211
195
212 var qName = m_context.Resolve(name);
196 var qName = m_context.Resolve(name);
213 var index = Array.FindIndex(m_attributes, x => x.QName == qName);
197 var index = Array.FindIndex(m_attributes, x => x.QName == qName);
214 if (index >= 0) {
198 if (index >= 0) {
215 MoveToAttributeImpl(index);
199 MoveToAttributeImpl(index);
216 return true;
200 return true;
217 }
201 }
218 return false;
202 return false;
219 }
203 }
220
204
221 public override bool MoveToAttribute(string name, string ns) {
205 public override bool MoveToAttribute(string name, string ns) {
222 if (m_attributes == null || m_attributes.Length == 0)
206 if (m_attributes == null || m_attributes.Length == 0)
223 return false;
207 return false;
224
208
225 var qName = m_context.Resolve(name);
209 var qName = m_context.Resolve(name);
226 var index = Array.FindIndex(m_attributes, x => x.QName == qName);
210 var index = Array.FindIndex(m_attributes, x => x.QName == qName);
227 if (index >= 0) {
211 if (index >= 0) {
228 MoveToAttributeImpl(index);
212 MoveToAttributeImpl(index);
229 return true;
213 return true;
230 }
214 }
231 return false;
215 return false;
232 }
216 }
233
217
234 void MoveToAttributeImpl(int i) {
218 void MoveToAttributeImpl(int i) {
235 if (!m_isAttribute) {
219 if (!m_isAttribute) {
236 m_elementQName = m_qName;
220 m_elementQName = m_qName;
237 m_elementDepth = m_xmlDepth;
221 m_elementDepth = m_xmlDepth;
238 m_elementPrefix = m_prefix;
222 m_elementPrefix = m_prefix;
239 m_elementIsEmpty = m_isEmpty;
223 m_elementIsEmpty = m_isEmpty;
240 m_isAttribute = true;
224 m_isAttribute = true;
241 }
225 }
242
226
243 var attr = m_attributes[i];
227 var attr = m_attributes[i];
244
228
245
229
246 m_currentAttribute = i;
230 m_currentAttribute = i;
247 m_currentAttributeRead = false;
231 m_currentAttributeRead = false;
248 m_nodeType = XmlNodeType.Attribute;
232 m_nodeType = XmlNodeType.Attribute;
249
233
250 m_xmlDepth = m_elementDepth + 1;
234 m_xmlDepth = m_elementDepth + 1;
251 m_qName = attr.QName;
235 m_qName = attr.QName;
252 m_value = attr.Value;
236 m_value = attr.Value;
253 m_prefix = attr.Prefix;
237 m_prefix = attr.Prefix;
254 }
238 }
255
239
256 public override bool MoveToElement() {
240 public override bool MoveToElement() {
257 if (m_isAttribute) {
241 if (m_isAttribute) {
258 m_value = null;
242 m_value = null;
259 m_nodeType = XmlNodeType.Element;
243 m_nodeType = XmlNodeType.Element;
260 m_xmlDepth = m_elementDepth;
244 m_xmlDepth = m_elementDepth;
261 m_prefix = m_elementPrefix;
245 m_prefix = m_elementPrefix;
262 m_qName = m_elementQName;
246 m_qName = m_elementQName;
263 m_isEmpty = m_elementIsEmpty;
247 m_isEmpty = m_elementIsEmpty;
264 m_isAttribute = false;
248 m_isAttribute = false;
265 return true;
249 return true;
266 }
250 }
267 return false;
251 return false;
268 }
252 }
269
253
270 public override bool MoveToFirstAttribute() {
254 public override bool MoveToFirstAttribute() {
271 if (m_attributes != null && m_attributes.Length > 0) {
255 if (m_attributes != null && m_attributes.Length > 0) {
272 MoveToAttributeImpl(0);
256 MoveToAttributeImpl(0);
273 return true;
257 return true;
274 }
258 }
275 return false;
259 return false;
276 }
260 }
277
261
278 public override bool MoveToNextAttribute() {
262 public override bool MoveToNextAttribute() {
279 if (m_isAttribute) {
263 if (m_isAttribute) {
280 var next = m_currentAttribute + 1;
264 var next = m_currentAttribute + 1;
281 if (next < AttributeCount) {
265 if (next < AttributeCount) {
282 MoveToAttributeImpl(next);
266 MoveToAttributeImpl(next);
283 return true;
267 return true;
284 }
268 }
285 return false;
269 return false;
286 } else {
270 } else {
287 return MoveToFirstAttribute();
271 return MoveToFirstAttribute();
288 }
272 }
289
273
290 }
274 }
291
275
292 public override bool ReadAttributeValue() {
276 public override bool ReadAttributeValue() {
293 if (!m_isAttribute || m_currentAttributeRead)
277 if (!m_isAttribute || m_currentAttributeRead)
294 return false;
278 return false;
295
279
296 ValueNode(m_attributes[m_currentAttribute].Value);
280 ValueNode(m_attributes[m_currentAttribute].Value);
297 m_currentAttributeRead = true;
281 m_currentAttributeRead = true;
298 return true;
282 return true;
299 }
283 }
300
284
301 public override void ResolveEntity() {
285 public override void ResolveEntity() {
302 /* do nothing */
286 /* do nothing */
303 }
287 }
304
288
305 /// <summary>
289 /// <summary>
306 /// Determines do we need to increase depth after the current node
290 /// Determines do we need to increase depth after the current node
307 /// </summary>
291 /// </summary>
308 /// <returns></returns>
292 /// <returns></returns>
309 public bool IsSibling() {
293 public bool IsSibling() {
310 switch (m_nodeType) {
294 switch (m_nodeType) {
311 case XmlNodeType.None: // start document
295 case XmlNodeType.None: // start document
312 case XmlNodeType.Attribute: // after attribute only it's content can be iterated with ReadAttributeValue method
296 case XmlNodeType.Attribute: // after attribute only it's content can be iterated with ReadAttributeValue method
313 return false;
297 return false;
314 case XmlNodeType.Element:
298 case XmlNodeType.Element:
315 // if the elemnt is empty the next element will be it's sibling
299 // if the elemnt is empty the next element will be it's sibling
316 return m_isEmpty;
300 return m_isEmpty;
317 default:
301 default:
318 return true;
302 return true;
319 }
303 }
320 }
304 }
321
305
322 void ValueNode(object value) {
306 void ValueNode(string value) {
323 if (!IsSibling()) // the node is nested
307 if (!IsSibling()) // the node is nested
324 m_xmlDepth++;
308 m_xmlDepth++;
325
309
326 m_qName = XmlQualifiedName.Empty;
310 m_qName = XmlQualifiedName.Empty;
327 m_nodeType = XmlNodeType.Text;
311 m_nodeType = XmlNodeType.Text;
328 m_prefix = string.Empty;
312 m_prefix = string.Empty;
329 m_value = value;
313 m_value = value;
330 m_isEmpty = false;
314 m_isEmpty = false;
331 m_attributes = null;
315 m_attributes = null;
332 }
316 }
333
317
334 void ElementNode(string name, string ns, XmlSimpleAttribute[] attrs, bool empty) {
318 void ElementNode(string name, string ns, XmlSimpleAttribute[] attrs, bool empty) {
335 if (!IsSibling()) // the node is nested
319 if (!IsSibling()) // the node is nested
336 m_xmlDepth++;
320 m_xmlDepth++;
337
321
338 var context = m_context;
322 var context = m_context;
339 List<XmlSimpleAttribute> definedAttrs = null;
323 List<XmlSimpleAttribute> definedAttrs = null;
340
324
341 // define new namespaces
325 // define new namespaces
342 if (attrs != null) {
326 if (attrs != null) {
343 foreach (var attr in attrs) {
327 foreach (var attr in attrs) {
344 if (attr.QName.Name == "xmlns") {
328 if (attr.QName.Name == "xmlns") {
345 if (context == m_context)
329 if (context == m_context)
346 context = new XmlNameContext(m_context, m_xmlDepth);
330 context = new XmlNameContext(m_context, m_xmlDepth);
347 context.DefinePrefix(ConvertValueToString(attr.Value), string.Empty);
331 context.DefinePrefix(attr.Value, string.Empty);
348 } else if (attr.Prefix == m_xmlnsPrefix) {
332 } else if (attr.Prefix == m_xmlnsPrefix) {
349 if (context == m_context)
333 if (context == m_context)
350 context = new XmlNameContext(m_context, m_xmlDepth);
334 context = new XmlNameContext(m_context, m_xmlDepth);
351 context.DefinePrefix(ConvertValueToString(attr.Value), attr.QName.Name);
335 context.DefinePrefix(attr.Value, attr.QName.Name);
352 } else {
336 } else {
353 string attrPrefix;
337 string attrPrefix;
354 if (string.IsNullOrEmpty(attr.QName.Namespace))
338 if (string.IsNullOrEmpty(attr.QName.Namespace))
355 continue;
339 continue;
356
340
357 // auto-define prefixes
341 // auto-define prefixes
358 if (!context.LookupNamespacePrefix(attr.QName.Namespace, out attrPrefix) || string.IsNullOrEmpty(attrPrefix)) {
342 if (!context.LookupNamespacePrefix(attr.QName.Namespace, out attrPrefix) || string.IsNullOrEmpty(attrPrefix)) {
359 // new namespace prefix added
343 // new namespace prefix added
360 attrPrefix = context.CreateNamespacePrefix(attr.QName.Namespace);
344 attrPrefix = context.CreateNamespacePrefix(attr.QName.Namespace);
361 attr.Prefix = attrPrefix;
345 attr.Prefix = attrPrefix;
362
346
363 if (definedAttrs == null)
347 if (definedAttrs == null)
364 definedAttrs = new List<XmlSimpleAttribute>();
348 definedAttrs = new List<XmlSimpleAttribute>();
365
349
366 definedAttrs.Add(new XmlSimpleAttribute(attrPrefix, m_xmlnsNamespace, m_xmlnsPrefix, attr.QName.Namespace));
350 definedAttrs.Add(new XmlSimpleAttribute(attrPrefix, m_xmlnsNamespace, m_xmlnsPrefix, attr.QName.Namespace));
367 }
351 }
368 }
352 }
369 }
353 }
370 }
354 }
371
355
372 string p;
356 string p;
373 // auto-define prefixes
357 // auto-define prefixes
374 if (!context.LookupNamespacePrefix(ns, out p)) {
358 if (!context.LookupNamespacePrefix(ns, out p)) {
375 if (context == m_context)
359 if (context == m_context)
376 context = new XmlNameContext(m_context, m_xmlDepth);
360 context = new XmlNameContext(m_context, m_xmlDepth);
377 p = context.CreateNamespacePrefix(ns);
361 p = context.CreateNamespacePrefix(ns);
378 if (definedAttrs == null)
362 if (definedAttrs == null)
379 definedAttrs = new List<XmlSimpleAttribute>();
363 definedAttrs = new List<XmlSimpleAttribute>();
380
364
381 definedAttrs.Add(new XmlSimpleAttribute(p, m_xmlnsNamespace, m_xmlnsPrefix, ns));
365 definedAttrs.Add(new XmlSimpleAttribute(p, m_xmlnsNamespace, m_xmlnsPrefix, ns));
382 }
366 }
383
367
384 if (definedAttrs != null) {
368 if (definedAttrs != null) {
385 if (attrs != null)
369 if (attrs != null)
386 definedAttrs.AddRange(attrs);
370 definedAttrs.AddRange(attrs);
387 attrs = definedAttrs.ToArray();
371 attrs = definedAttrs.ToArray();
388 }
372 }
389
373
390 if (!empty)
374 if (!empty)
391 m_context = context;
375 m_context = context;
392
376
393 m_nodeType = XmlNodeType.Element;
377 m_nodeType = XmlNodeType.Element;
394 m_qName = new XmlQualifiedName(name, ns);
378 m_qName = new XmlQualifiedName(name, ns);
395 m_prefix = p;
379 m_prefix = p;
396 m_value = null;
380 m_value = null;
397 m_isEmpty = empty;
381 m_isEmpty = empty;
398 m_attributes = attrs;
382 m_attributes = attrs;
399 }
383 }
400
384
401 void EndElementNode(string name, string ns) {
385 void EndElementNode(string name, string ns) {
402 if (IsSibling()) {
386 if (IsSibling()) {
403 // closing the element which has children
387 // closing the element which has children
404 m_xmlDepth--;
388 m_xmlDepth--;
405 }
389 }
406
390
407 string p;
391 string p;
408 if (!m_context.LookupNamespacePrefix(ns, out p))
392 if (!m_context.LookupNamespacePrefix(ns, out p))
409 throw new Exception($"Failed to lookup namespace '{ns}'");
393 throw new Exception($"Failed to lookup namespace '{ns}'");
410
394
411 if (m_context.Depth == m_xmlDepth)
395 if (m_context.Depth == m_xmlDepth)
412 m_context = m_context.ParentContext;
396 m_context = m_context.ParentContext;
413
397
414 m_nodeType = XmlNodeType.EndElement;
398 m_nodeType = XmlNodeType.EndElement;
415 m_prefix = p;
399 m_prefix = p;
416 m_qName = new XmlQualifiedName(name, ns);
400 m_qName = new XmlQualifiedName(name, ns);
417 m_value = null;
401 m_value = null;
418 m_attributes = null;
402 m_attributes = null;
419 m_isEmpty = false;
403 m_isEmpty = false;
420 }
404 }
421
405
422 void XmlDeclaration() {
406 void XmlDeclaration() {
423 if (!IsSibling()) // the node is nested
407 if (!IsSibling()) // the node is nested
424 m_xmlDepth++;
408 m_xmlDepth++;
425 m_nodeType = XmlNodeType.XmlDeclaration;
409 m_nodeType = XmlNodeType.XmlDeclaration;
426 m_qName = new XmlQualifiedName("xml");
410 m_qName = new XmlQualifiedName("xml");
427 m_value = "version='1.0'";
411 m_value = "version='1.0'";
428 m_prefix = string.Empty;
412 m_prefix = string.Empty;
429 m_attributes = null;
413 m_attributes = null;
430 m_isEmpty = false;
414 m_isEmpty = false;
431 }
415 }
432
416
433 public override bool Read() {
417 public override bool Read() {
434 try {
418 try {
435 string elementName;
419 string elementName;
436 XmlSimpleAttribute[] elementAttrs = null;
420 XmlSimpleAttribute[] elementAttrs = null;
437 MoveToElement();
421 MoveToElement();
438
422
439 switch (m_position) {
423 switch (m_position) {
440 case JsonXmlReaderPosition.Initial:
424 case JsonXmlReaderPosition.Initial:
441 m_jsonLocalName = m_jsonRootName;
425 m_jsonLocalName = m_jsonRootName;
442 m_jsonSkip = false;
426 m_jsonSkip = false;
443 XmlDeclaration();
427 XmlDeclaration();
444 m_position = JsonXmlReaderPosition.Declaration;
428 m_position = JsonXmlReaderPosition.Declaration;
445 return true;
429 return true;
446 case JsonXmlReaderPosition.Declaration:
430 case JsonXmlReaderPosition.Declaration:
447 elementAttrs = new[] {
431 elementAttrs = new[] {
448 new XmlSimpleAttribute(m_xsiPrefix, m_xmlnsNamespace, m_xmlnsPrefix, m_xsiNamespace),
432 new XmlSimpleAttribute(m_xsiPrefix, m_xmlnsNamespace, m_xmlnsPrefix, m_xsiNamespace),
449 string.IsNullOrEmpty(m_jsonPrefix) ?
433 string.IsNullOrEmpty(m_jsonPrefix) ?
450 new XmlSimpleAttribute(m_xmlnsPrefix, string.Empty, string.Empty, m_jsonNamespace) :
434 new XmlSimpleAttribute(m_xmlnsPrefix, string.Empty, string.Empty, m_jsonNamespace) :
451 new XmlSimpleAttribute(m_jsonPrefix, m_xmlnsNamespace, m_xmlnsPrefix, m_jsonNamespace)
435 new XmlSimpleAttribute(m_jsonPrefix, m_xmlnsNamespace, m_xmlnsPrefix, m_jsonNamespace)
452 };
436 };
453 break;
437 break;
454 case JsonXmlReaderPosition.ValueElement:
438 case JsonXmlReaderPosition.ValueElement:
455 if (!m_isEmpty) {
439 if (!m_isEmpty) {
456 if (m_parser.ElementValue != null && !m_parser.ElementValue.Equals(string.Empty))
440 if (m_parser.ElementValue != null && !m_parser.ElementValue.Equals(string.Empty))
457 ValueNode(m_parser.ElementValue);
441 ValueNode(m_parser.ElementValue);
458 else
442 else
459 goto case JsonXmlReaderPosition.ValueContent;
443 goto case JsonXmlReaderPosition.ValueContent;
460 m_position = JsonXmlReaderPosition.ValueContent;
444 m_position = JsonXmlReaderPosition.ValueContent;
461 return true;
445 return true;
462 } else {
446 } else {
463 m_position = JsonXmlReaderPosition.ValueEndElement;
447 m_position = JsonXmlReaderPosition.ValueEndElement;
464 break;
448 break;
465 }
449 }
466 case JsonXmlReaderPosition.ValueContent:
450 case JsonXmlReaderPosition.ValueContent:
467 EndElementNode(m_jsonValueName, m_jsonNamespace);
451 EndElementNode(m_jsonValueName, m_jsonNamespace);
468 m_position = JsonXmlReaderPosition.ValueEndElement;
452 m_position = JsonXmlReaderPosition.ValueEndElement;
469 return true;
453 return true;
470 case JsonXmlReaderPosition.Eof:
454 case JsonXmlReaderPosition.Eof:
471 case JsonXmlReaderPosition.Closed:
455 case JsonXmlReaderPosition.Closed:
472 case JsonXmlReaderPosition.Error:
456 case JsonXmlReaderPosition.Error:
473 return false;
457 return false;
474 }
458 }
475
459
476 while (m_parser.Read()) {
460 while (m_parser.Read()) {
477 var jsonName = m_nameTable.Add(m_parser.ElementName);
461 var jsonName = m_nameTable.Add(m_parser.ElementName);
478
462
479 switch (m_parser.ElementType) {
463 switch (m_parser.ElementType) {
480 case JsonElementType.BeginObject:
464 case JsonElementType.BeginObject:
481 if (!EnterJsonObject(jsonName, out elementName))
465 if (!EnterJsonObject(jsonName, out elementName))
482 continue;
466 continue;
483
467
484 m_position = JsonXmlReaderPosition.BeginObject;
468 m_position = JsonXmlReaderPosition.BeginObject;
485 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
469 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
486 break;
470 break;
487 case JsonElementType.EndObject:
471 case JsonElementType.EndObject:
488 if (!LeaveJsonScope(out elementName))
472 if (!LeaveJsonScope(out elementName))
489 continue;
473 continue;
490
474
491 m_position = JsonXmlReaderPosition.EndObject;
475 m_position = JsonXmlReaderPosition.EndObject;
492 EndElementNode(elementName, m_jsonNamespace);
476 EndElementNode(elementName, m_jsonNamespace);
493 break;
477 break;
494 case JsonElementType.BeginArray:
478 case JsonElementType.BeginArray:
495 if (!EnterJsonArray(jsonName, out elementName))
479 if (!EnterJsonArray(jsonName, out elementName))
496 continue;
480 continue;
497
481
498 m_position = JsonXmlReaderPosition.BeginArray;
482 m_position = JsonXmlReaderPosition.BeginArray;
499 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
483 ElementNode(elementName, m_jsonNamespace, elementAttrs, false);
500 break;
484 break;
501 case JsonElementType.EndArray:
485 case JsonElementType.EndArray:
502 if (!LeaveJsonScope(out elementName))
486 if (!LeaveJsonScope(out elementName))
503 continue;
487 continue;
504
488
505 m_position = JsonXmlReaderPosition.EndArray;
489 m_position = JsonXmlReaderPosition.EndArray;
506 EndElementNode(elementName, m_jsonNamespace);
490 EndElementNode(elementName, m_jsonNamespace);
507 break;
491 break;
508 case JsonElementType.Value:
492 case JsonElementType.Value:
509 if (!VisitJsonValue(jsonName, out m_jsonValueName))
493 if (!VisitJsonValue(jsonName, out m_jsonValueName))
510 continue;
494 continue;
511
495
512 m_position = JsonXmlReaderPosition.ValueElement;
496 m_position = JsonXmlReaderPosition.ValueElement;
513 if (m_parser.ElementValue == null)
497 if (m_parser.ElementValue == null)
514 // generate empty element with xsi:nil="true" attribute
498 // generate empty element with xsi:nil="true" attribute
515 ElementNode(
499 ElementNode(
516 m_jsonValueName,
500 m_jsonValueName,
517 m_jsonNamespace,
501 m_jsonNamespace,
518 new[] {
502 new[] {
519 new XmlSimpleAttribute("nil", m_xsiNamespace, m_xsiPrefix, true)
503 new XmlSimpleAttribute("nil", m_xsiNamespace, m_xsiPrefix, "true")
520 },
504 },
521 true
505 true
522 );
506 );
523 else
507 else
524 ElementNode(m_jsonValueName, m_jsonNamespace, elementAttrs, m_parser.ElementValue.Equals(string.Empty));
508 ElementNode(m_jsonValueName, m_jsonNamespace, elementAttrs, m_parser.ElementValue.Equals(string.Empty));
525 break;
509 break;
526 default:
510 default:
527 throw new Exception($"Unexpected JSON element {m_parser.ElementType}: {m_parser.ElementName}");
511 throw new Exception($"Unexpected JSON element {m_parser.ElementType}: {m_parser.ElementName}");
528 }
512 }
529 return true;
513 return true;
530 }
514 }
531
515
532 m_position = JsonXmlReaderPosition.Eof;
516 m_position = JsonXmlReaderPosition.Eof;
533 return false;
517 return false;
534 } catch {
518 } catch {
535 m_position = JsonXmlReaderPosition.Error;
519 m_position = JsonXmlReaderPosition.Error;
536 throw;
520 throw;
537 }
521 }
538 }
522 }
539
523
540 void SaveJsonName() {
524 void SaveJsonName() {
541 m_jsonNameStack.Push(new JsonContext {
525 m_jsonNameStack.Push(new JsonContext {
542 skip = m_jsonSkip,
526 skip = m_jsonSkip,
543 localName = m_jsonLocalName
527 localName = m_jsonLocalName
544 });
528 });
545
529
546 }
530 }
547
531
548 bool EnterJsonObject(string name, out string elementName) {
532 bool EnterJsonObject(string name, out string elementName) {
549 SaveJsonName();
533 SaveJsonName();
550 m_jsonSkip = false;
534 m_jsonSkip = false;
551
535
552 if (string.IsNullOrEmpty(name)) {
536 if (string.IsNullOrEmpty(name)) {
553 if (m_jsonNameStack.Count != 1 && !m_jsonFlattenArrays)
537 if (m_jsonNameStack.Count != 1 && !m_jsonFlattenArrays)
554 m_jsonLocalName = m_jsonArrayItemName;
538 m_jsonLocalName = m_jsonArrayItemName;
555 } else {
539 } else {
556 m_jsonLocalName = name;
540 m_jsonLocalName = name;
557 }
541 }
558
542
559 elementName = m_jsonLocalName;
543 elementName = m_jsonLocalName;
560 return true;
544 return true;
561 }
545 }
562
546
563 /// <summary>
547 /// <summary>
564 /// Called when JSON parser visits BeginArray ('[') element.
548 /// Called when JSON parser visits BeginArray ('[') element.
565 /// </summary>
549 /// </summary>
566 /// <param name="name">Optional property name if the array is the member of an object</param>
550 /// <param name="name">Optional property name if the array is the member of an object</param>
567 /// <returns>true if element should be emited, false otherwise</returns>
551 /// <returns>true if element should be emited, false otherwise</returns>
568 bool EnterJsonArray(string name, out string elementName) {
552 bool EnterJsonArray(string name, out string elementName) {
569 SaveJsonName();
553 SaveJsonName();
570
554
571 if (string.IsNullOrEmpty(name)) {
555 if (string.IsNullOrEmpty(name)) {
572 // m_jsonNameStack.Count == 1 means the root node
556 // m_jsonNameStack.Count == 1 means the root node
573 if (m_jsonNameStack.Count != 1 && !m_jsonFlattenArrays)
557 if (m_jsonNameStack.Count != 1 && !m_jsonFlattenArrays)
574 m_jsonLocalName = m_jsonArrayItemName;
558 m_jsonLocalName = m_jsonArrayItemName;
575
559
576 m_jsonSkip = false; // we should not flatten arrays inside arrays or in the document root
560 m_jsonSkip = false; // we should not flatten arrays inside arrays or in the document root
577 } else {
561 } else {
578 m_jsonLocalName = name;
562 m_jsonLocalName = name;
579 m_jsonSkip = m_jsonFlattenArrays;
563 m_jsonSkip = m_jsonFlattenArrays;
580 }
564 }
581 elementName = m_jsonLocalName;
565 elementName = m_jsonLocalName;
582
566
583 return !m_jsonSkip;
567 return !m_jsonSkip;
584 }
568 }
585
569
586 bool VisitJsonValue(string name, out string elementName) {
570 bool VisitJsonValue(string name, out string elementName) {
587 if (string.IsNullOrEmpty(name)) {
571 if (string.IsNullOrEmpty(name)) {
588 // m_jsonNameStack.Count == 0 means that JSON document consists from simple value
572 // m_jsonNameStack.Count == 0 means that JSON document consists from simple value
589 elementName = (m_jsonNameStack.Count == 0 || m_jsonFlattenArrays) ? m_jsonLocalName : m_jsonArrayItemName;
573 elementName = (m_jsonNameStack.Count == 0 || m_jsonFlattenArrays) ? m_jsonLocalName : m_jsonArrayItemName;
590 } else {
574 } else {
591 elementName = name;
575 elementName = name;
592 }
576 }
593 return true;
577 return true;
594 }
578 }
595
579
596 bool LeaveJsonScope(out string elementName) {
580 bool LeaveJsonScope(out string elementName) {
597 elementName = m_jsonLocalName;
581 elementName = m_jsonLocalName;
598 var skip = m_jsonSkip;
582 var skip = m_jsonSkip;
599
583
600 var prev = m_jsonNameStack.Pop();
584 var prev = m_jsonNameStack.Pop();
601 m_jsonLocalName = prev.localName;
585 m_jsonLocalName = prev.localName;
602 m_jsonSkip = prev.skip;
586 m_jsonSkip = prev.skip;
603
587
604 return !skip;
588 return !skip;
605 }
589 }
606
590
607 public override string ToString() {
591 public override string ToString() {
608 switch (NodeType) {
592 switch (NodeType) {
609 case XmlNodeType.Element:
593 case XmlNodeType.Element:
610 return $"<{Name} {string.Join(" ", (m_attributes ?? new XmlSimpleAttribute[0]).Select(x => $"{x.Prefix}{(string.IsNullOrEmpty(x.Prefix) ? "" : ":")}{x.QName.Name}='{ConvertValueToString(x.Value)}'"))} {(IsEmptyElement ? "/" : "")}>";
594 return $"<{Name} {string.Join(" ", (m_attributes ?? new XmlSimpleAttribute[0]).Select(x => $"{x.Prefix}{(string.IsNullOrEmpty(x.Prefix) ? "" : ":")}{x.QName.Name}='{x.Value}'"))} {(IsEmptyElement ? "/" : "")}>";
611 case XmlNodeType.Attribute:
595 case XmlNodeType.Attribute:
612 return $"@{Name}";
596 return $"@{Name}";
613 case XmlNodeType.Text:
597 case XmlNodeType.Text:
614 return $"{Value}";
598 return $"{Value}";
615 case XmlNodeType.CDATA:
599 case XmlNodeType.CDATA:
616 return $"<![CDATA[{Value}]]>";
600 return $"<![CDATA[{Value}]]>";
617 case XmlNodeType.EntityReference:
601 case XmlNodeType.EntityReference:
618 return $"&{Name};";
602 return $"&{Name};";
619 case XmlNodeType.EndElement:
603 case XmlNodeType.EndElement:
620 return $"</{Name}>";
604 return $"</{Name}>";
621 default:
605 default:
622 return $".{NodeType} {Name} {Value}";
606 return $".{NodeType} {Name} {Value}";
623 }
607 }
624 }
608 }
625 }
609 }
626 }
610 }
@@ -1,22 +1,22
1 using System;
1 using System;
2 using System.Collections.Generic;
2 using System.Collections.Generic;
3 using System.Linq;
3 using System.Linq;
4 using System.Text;
4 using System.Text;
5 using System.Threading.Tasks;
5 using System.Threading.Tasks;
6 using System.Xml;
6 using System.Xml;
7
7
8 namespace Implab.Xml {
8 namespace Implab.Xml {
9 public class XmlSimpleAttribute {
9 public class XmlSimpleAttribute {
10 public XmlSimpleAttribute(string name, string ns, string prefix, object value) {
10 public XmlSimpleAttribute(string name, string ns, string prefix, string value) {
11 QName = new XmlQualifiedName(name, ns);
11 QName = new XmlQualifiedName(name, ns);
12 Prefix = prefix;
12 Prefix = prefix;
13 Value = value;
13 Value = value;
14 }
14 }
15
15
16 public XmlQualifiedName QName { get; set; }
16 public XmlQualifiedName QName { get; set; }
17
17
18 public string Prefix { get; set; }
18 public string Prefix { get; set; }
19
19
20 public object Value { get; set; }
20 public string Value { get; set; }
21 }
21 }
22 }
22 }
General Comments 3
Under Review
author

Auto status change to "Under Review"

Approved
author

ok, latest stable version should be in default

You need to be logged in to leave comments. Login now