##// END OF EJS Templates
rewritten the text scanner
cin -
r176:0c3c69fe225b ref20160224
parent child
Show More
@@ -0,0 +1,30
1 using System;
2 using System.IO;
3
4 namespace Implab.Formats {
5 public class ReaderScanner: TextScanner {
6 const int CHUNK_SIZE = 1024;
7 const int BUFFER_MAX = CHUNK_SIZE*1024;
8
9 readonly TextReader m_reader;
10
11 public ReaderScanner(TextReader reader, int limit, int chunk) : base(limit, chunk) {
12 Safe.ArgumentNotNull(reader, "reader");
13 m_reader = reader;
14 }
15
16 public ReaderScanner(TextReader reader) : this(reader, BUFFER_MAX, CHUNK_SIZE) {
17 }
18
19 protected override int Read(char[] buffer, int offset, int size) {
20 return m_reader.Read(buffer, offset, size);
21 }
22
23 protected override void Dispose(bool disposing) {
24 if (disposing)
25 Safe.Dispose(m_reader);
26 base.Dispose(disposing);
27 }
28 }
29 }
30
@@ -0,0 +1,24
1 using System;
2
3 namespace Implab.Formats {
4 public class ScannerContext<TTag> {
5 public int[,] Dfa { get; private set; }
6 public bool[] Final { get; private set; }
7 public TTag[][] Tags { get; private set; }
8 public int State { get; private set; }
9 public int[] Alphabet { get; private set; }
10
11 public ScannerContext(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet) {
12 Dfa = dfa;
13 Final = final;
14 Tags = tags;
15 State = state;
16 Alphabet = alphabet;
17 }
18
19 public bool Execute(TextScanner scanner, out TTag[] tag) {
20 return scanner.ReadToken(Dfa, Final, Tags, State, Alphabet, out tag);
21 }
22 }
23 }
24
@@ -0,0 +1,26
1 using System;
2
3 namespace Implab.Formats {
4 public class StringScanner: TextScanner {
5 const int CHUNK_SIZE = 1024;
6
7 readonly string m_text;
8 int m_pos;
9
10 public StringScanner(string text) : base(text.Length, text.Length < CHUNK_SIZE ? text.Length : CHUNK_SIZE) {
11 m_text = text;
12 Feed();
13 }
14
15 protected override int Read(char[] buffer, int offset, int size) {
16 var actual = size + m_pos > m_text.Length ? m_text.Length - m_pos : size;
17
18 m_text.CopyTo(m_pos,buffer,offset, actual);
19
20 m_pos += actual;
21
22 return actual;
23 }
24 }
25 }
26
This diff has been collapsed as it changes many lines, (618 lines changed) Show them Hide them
@@ -1,305 +1,313
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5
5
6 namespace Implab.Automaton {
6 namespace Implab.Automaton {
7 public class DFATable : IDFATableBuilder {
7 public class DFATable : IDFATableBuilder {
8 int m_stateCount;
8 int m_stateCount;
9 int m_symbolCount;
9 int m_symbolCount;
10 int m_initialState;
10 int m_initialState;
11
11
12 readonly HashSet<int> m_finalStates = new HashSet<int>();
12 readonly HashSet<int> m_finalStates = new HashSet<int>();
13 readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
13 readonly HashSet<AutomatonTransition> m_transitions = new HashSet<AutomatonTransition>();
14
14
15
15
16 #region IDFADefinition implementation
16 #region IDFADefinition implementation
17
17
18 public bool IsFinalState(int s) {
18 public bool IsFinalState(int s) {
19 Safe.ArgumentInRange(s, 0, m_stateCount, "s");
19 Safe.ArgumentInRange(s, 0, m_stateCount, "s");
20
20
21 return m_finalStates.Contains(s);
21 return m_finalStates.Contains(s);
22 }
22 }
23
23
24 public IEnumerable<int> FinalStates {
24 public IEnumerable<int> FinalStates {
25 get {
25 get {
26 return m_finalStates;
26 return m_finalStates;
27 }
27 }
28 }
28 }
29
29
30 public int StateCount {
30 public int StateCount {
31 get { return m_stateCount; }
31 get { return m_stateCount; }
32 }
32 }
33
33
34 public int AlphabetSize {
34 public int AlphabetSize {
35 get { return m_symbolCount; }
35 get { return m_symbolCount; }
36 }
36 }
37
37
38 public int InitialState {
38 public int InitialState {
39 get { return m_initialState; }
39 get { return m_initialState; }
40 }
40 }
41
41
42 #endregion
42 #endregion
43
43
44 public void SetInitialState(int s) {
44 public void SetInitialState(int s) {
45 Safe.ArgumentAssert(s >= 0, "s");
45 Safe.ArgumentAssert(s >= 0, "s");
46 m_initialState = s;
46 m_initialState = s;
47 }
47 }
48
48
49 public void MarkFinalState(int state) {
49 public void MarkFinalState(int state) {
50 m_finalStates.Add(state);
50 m_finalStates.Add(state);
51 }
51 }
52
52
53 public void Add(AutomatonTransition item) {
53 public void Add(AutomatonTransition item) {
54 Safe.ArgumentAssert(item.s1 >= 0, "item");
54 Safe.ArgumentAssert(item.s1 >= 0, "item");
55 Safe.ArgumentAssert(item.s2 >= 0, "item");
55 Safe.ArgumentAssert(item.s2 >= 0, "item");
56 Safe.ArgumentAssert(item.edge >= 0, "item");
56 Safe.ArgumentAssert(item.edge >= 0, "item");
57
57
58 m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
58 m_stateCount = Math.Max(m_stateCount, Math.Max(item.s1, item.s2) + 1);
59 m_symbolCount = Math.Max(m_symbolCount, item.edge);
59 m_symbolCount = Math.Max(m_symbolCount, item.edge);
60
60
61 m_transitions.Add(item);
61 m_transitions.Add(item);
62 }
62 }
63
63
64 public void Clear() {
64 public void Clear() {
65 m_stateCount = 0;
65 m_stateCount = 0;
66 m_symbolCount = 0;
66 m_symbolCount = 0;
67 m_finalStates.Clear();
67 m_finalStates.Clear();
68 m_transitions.Clear();
68 m_transitions.Clear();
69 }
69 }
70
70
71 public bool Contains(AutomatonTransition item) {
71 public bool Contains(AutomatonTransition item) {
72 return m_transitions.Contains(item);
72 return m_transitions.Contains(item);
73 }
73 }
74
74
75 public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
75 public void CopyTo(AutomatonTransition[] array, int arrayIndex) {
76 m_transitions.CopyTo(array, arrayIndex);
76 m_transitions.CopyTo(array, arrayIndex);
77 }
77 }
78
78
79 public bool Remove(AutomatonTransition item) {
79 public bool Remove(AutomatonTransition item) {
80 m_transitions.Remove(item);
80 m_transitions.Remove(item);
81 }
81 }
82
82
83 public int Count {
83 public int Count {
84 get {
84 get {
85 return m_transitions.Count;
85 return m_transitions.Count;
86 }
86 }
87 }
87 }
88
88
89 public bool IsReadOnly {
89 public bool IsReadOnly {
90 get {
90 get {
91 return false;
91 return false;
92 }
92 }
93 }
93 }
94
94
95 public IEnumerator<AutomatonTransition> GetEnumerator() {
95 public IEnumerator<AutomatonTransition> GetEnumerator() {
96 return m_transitions.GetEnumerator();
96 return m_transitions.GetEnumerator();
97 }
97 }
98
98
99 System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
99 System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() {
100 return GetEnumerator();
100 return GetEnumerator();
101 }
101 }
102
102
103 public DFAStateDescriptor[] CreateTransitionTable() {
103 public int[,] CreateTransitionTable() {
104 var table = new DFAStateDescriptor[StateCount];
104 var table = new int[StateCount,AlphabetSize];
105
105
106 foreach (var t in this) {
106 for (int i = 0; i < StateCount; i++)
107 if (table[t.s1].transitions == null)
107 for (int j = 0; i < AlphabetSize; j++)
108 table[t.s1] = new DFAStateDescriptor(AlphabetSize, IsFinalState(t.s1));
108 table[i, j] = DFAConst.UNREACHABLE_STATE;
109 if (table[t.s2].transitions == null)
109
110 table[t.s2] = new DFAStateDescriptor(AlphabetSize, IsFinalState(t.s2));
110 foreach (var t in this)
111 table[t.s1].transitions[t.edge] = t.s2;
111 table[t.s1,t.edge] = t.s2;
112 }
112
113
113 return table;
114 return table;
114 }
115 }
115
116
116 public bool[] CreateFinalStateTable() {
117 /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
117 var table = new bool[StateCount];
118 /// <remarks>
118
119 /// В процессе построения минимального автомата требуется разделить множество состояний,
119 foreach (var s in FinalStates)
120 /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
120 table[s] = true;
121 /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
121
122 /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
122 return table;
123 /// </remarks>
123 }
124 /// <returns>The final states.</returns>
124
125 protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
125 /// <summary>Формирует множества конечных состояний перед началом работы алгоритма минимизации.</summary>
126 return new HashSet<int>[] { m_finalStates };
126 /// <remarks>
127 }
127 /// В процессе построения минимального автомата требуется разделить множество состояний,
128
128 /// на два подмножества - конечные состояния и все остальные, после чего эти подмножества
129 protected void Optimize(
129 /// будут резделены на более мелкие. Иногда требуется гарантировать различия конечных сосотяний,
130 IDFATableBuilder optimalDFA,
130 /// для этого необходимо переопределить даннцю фукнцию, для получения множеств конечных состояний.
131 IDictionary<int,int> alphabetMap,
131 /// </remarks>
132 IDictionary<int,int> stateMap
132 /// <returns>The final states.</returns>
133 ) {
133 protected virtual IEnumerable<HashSet<int>> GroupFinalStates() {
134 Safe.ArgumentNotNull(optimalDFA, "dfa");
134 return new HashSet<int>[] { m_finalStates };
135 Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
135 }
136 Safe.ArgumentNotNull(stateMap, "stateMap");
136
137
137 protected void Optimize(
138
138 IDFATableBuilder optimalDFA,
139 var setComparer = new CustomEqualityComparer<HashSet<int>>(
139 IDictionary<int,int> alphabetMap,
140 (x, y) => x.SetEquals(y),
140 IDictionary<int,int> stateMap
141 s => s.Sum(x => x.GetHashCode())
141 ) {
142 );
142 Safe.ArgumentNotNull(optimalDFA, "dfa");
143
143 Safe.ArgumentNotNull(alphabetMap, "alphabetMap");
144 var optimalStates = new HashSet<HashSet<int>>(setComparer);
144 Safe.ArgumentNotNull(stateMap, "stateMap");
145 var queue = new HashSet<HashSet<int>>(setComparer);
145
146
146
147 // получаем конечные состояния, сгруппированные по маркерам
147 var setComparer = new CustomEqualityComparer<HashSet<int>>(
148 optimalStates.UnionWith(
148 (x, y) => x.SetEquals(y),
149 GroupFinalStates()
149 s => s.Sum(x => x.GetHashCode())
150 );
150 );
151
151
152 var state = new HashSet<int>(
152 var optimalStates = new HashSet<HashSet<int>>(setComparer);
153 Enumerable
153 var queue = new HashSet<HashSet<int>>(setComparer);
154 .Range(0, m_stateCount - 1)
154
155 .Where(i => !m_finalStates.Contains(i))
155 // получаем конечные состояния, сгруппированные по маркерам
156 );
156 optimalStates.UnionWith(
157
157 GroupFinalStates()
158 optimalStates.Add(state);
158 );
159 queue.Add(state);
159
160
160 var state = new HashSet<int>(
161 var rmap = m_transitions
161 Enumerable
162 .GroupBy(t => t.s2)
162 .Range(0, m_stateCount - 1)
163 .ToLookup(
163 .Where(i => !m_finalStates.Contains(i))
164 g => g.Key, // s2
164 );
165 g => g.ToLookup(t => t.edge, t => t.s1)
165
166 );
166 optimalStates.Add(state);
167
167 queue.Add(state);
168 while (queue.Count > 0) {
168
169 var stateA = queue.First();
169 var rmap = m_transitions
170 queue.Remove(stateA);
170 .GroupBy(t => t.s2)
171
171 .ToLookup(
172 for (int c = 0; c < m_symbolCount; c++) {
172 g => g.Key, // s2
173 var stateX = new HashSet<int>();
173 g => g.ToLookup(t => t.edge, t => t.s1)
174 foreach(var a in stateA)
174 );
175 stateX.UnionWith(rmap[a][c]); // all states from wich 'c' leads to 'a'
175
176
176 while (queue.Count > 0) {
177 foreach (var stateY in optimalStates.ToArray()) {
177 var stateA = queue.First();
178 if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
178 queue.Remove(stateA);
179 var stateR1 = new HashSet<int>(stateY);
179
180 var stateR2 = new HashSet<int>(stateY);
180 for (int c = 0; c < m_symbolCount; c++) {
181
181 var stateX = new HashSet<int>();
182 stateR1.IntersectWith(stateX);
182 foreach(var a in stateA)
183 stateR2.ExceptWith(stateX);
183 stateX.UnionWith(rmap[a][c]); // all states from wich 'c' leads to 'a'
184
184
185 optimalStates.Remove(stateY);
185 foreach (var stateY in optimalStates.ToArray()) {
186 optimalStates.Add(stateR1);
186 if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
187 optimalStates.Add(stateR2);
187 var stateR1 = new HashSet<int>(stateY);
188
188 var stateR2 = new HashSet<int>(stateY);
189 if (queue.Contains(stateY)) {
189
190 queue.Remove(stateY);
190 stateR1.IntersectWith(stateX);
191 queue.Add(stateR1);
191 stateR2.ExceptWith(stateX);
192 queue.Add(stateR2);
192
193 } else {
193 optimalStates.Remove(stateY);
194 queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
194 optimalStates.Add(stateR1);
195 }
195 optimalStates.Add(stateR2);
196 }
196
197 }
197 if (queue.Contains(stateY)) {
198 }
198 queue.Remove(stateY);
199 }
199 queue.Add(stateR1);
200
200 queue.Add(stateR2);
201 // карта получения оптимального состояния по соотвествующему ему простому состоянию
201 } else {
202 var nextState = 0;
202 queue.Add(stateR1.Count <= stateR2.Count ? stateR1 : stateR2);
203 foreach (var item in optimalStates) {
203 }
204 var id = nextState++;
204 }
205 foreach (var s in item)
205 }
206 stateMap[s] = id;
206 }
207 }
207 }
208
208
209 // получаем минимальный алфавит
209 // карта получения оптимального состояния по соотвествующему ему простому состоянию
210 // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
210 var nextState = 0;
211 // для этого используем алгоритм кластеризации, сначала
211 foreach (var item in optimalStates) {
212 // считаем, что все символы не различимы
212 var id = nextState++;
213
213 foreach (var s in item)
214 var minClasses = new HashSet<HashSet<int>>(setComparer);
214 stateMap[s] = id;
215 var alphaQueue = new Queue<HashSet<int>>();
215 }
216 alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
216
217
217 // получаем минимальный алфавит
218 // для всех состояний, будем проверять каждый класс на различимость,
218 // входные символы не различимы, если Move(s,a1) == Move(s,a2), для любого s
219 // т.е. символы различимы, если они приводят к разным состояниям
219 // для этого используем алгоритм кластеризации, сначала
220 for (int s = 0 ; s < optimalStates.Count; s++) {
220 // считаем, что все символы не различимы
221 var newQueue = new Queue<HashSet<int>>();
221
222
222 var minClasses = new HashSet<HashSet<int>>(setComparer);
223 foreach (var A in alphaQueue) {
223 var alphaQueue = new Queue<HashSet<int>>();
224 // классы из одного символа делить бесполезно, переводим их сразу в
224 alphaQueue.Enqueue(new HashSet<int>(Enumerable.Range(0,AlphabetSize)));
225 // результирующий алфавит
225
226 if (A.Count == 1) {
226 // для всех состояний, будем проверять каждый класс на различимость,
227 minClasses.Add(A);
227 // т.е. символы различимы, если они приводят к разным состояниям
228 continue;
228 for (int s = 0 ; s < optimalStates.Count; s++) {
229 }
229 var newQueue = new Queue<HashSet<int>>();
230
230
231 // различаем классы символов, которые переводят в различные оптимальные состояния
231 foreach (var A in alphaQueue) {
232 // optimalState -> alphaClass
232 // классы из одного символа делить бесполезно, переводим их сразу в
233 var classes = new Dictionary<int, HashSet<int>>();
233 // результирующий алфавит
234
234 if (A.Count == 1) {
235 foreach (var term in A) {
235 minClasses.Add(A);
236 // ищем все переходы класса по символу term
236 continue;
237 var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray();
237 }
238
238
239 var s2 = res.Length > 0 ? res[0] : -1;
239 // различаем классы символов, которые переводят в различные оптимальные состояния
240
240 // optimalState -> alphaClass
241 HashSet<int> a2;
241 var classes = new Dictionary<int, HashSet<int>>();
242 if (!classes.TryGetValue(s2, out a2)) {
242
243 a2 = new HashSet<int>();
243 foreach (var term in A) {
244 newQueue.Enqueue(a2);
244 // ищем все переходы класса по символу term
245 classes[s2] = a2;
245 var res = m_transitions.Where(t => stateMap[t.s1] == s && t.edge == term).Select(t => stateMap[t.s2]).ToArray();
246 }
246
247 a2.Add(term);
247 var s2 = res.Length > 0 ? res[0] : -1;
248 }
248
249 }
249 HashSet<int> a2;
250
250 if (!classes.TryGetValue(s2, out a2)) {
251 if (newQueue.Count == 0)
251 a2 = new HashSet<int>();
252 break;
252 newQueue.Enqueue(a2);
253 alphaQueue = newQueue;
253 classes[s2] = a2;
254 }
254 }
255
255 a2.Add(term);
256 // после окончания работы алгоритма в очереди останутся минимальные различимые классы
256 }
257 // входных символов
257 }
258 foreach (var A in alphaQueue)
258
259 minClasses.Add(A);
259 if (newQueue.Count == 0)
260
260 break;
261 // построение отображения алфавитов входных символов.
261 alphaQueue = newQueue;
262 // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
262 }
263 // специальное значение, тогда сохраним минимальный класс,
263
264 // содержащий этот символ на томже месте.
264 // после окончания работы алгоритма в очереди останутся минимальные различимые классы
265
265 // входных символов
266 var nextCls = 0;
266 foreach (var A in alphaQueue)
267 foreach (var item in minClasses) {
267 minClasses.Add(A);
268 if (nextCls == DFAConst.UNCLASSIFIED_INPUT)
268
269 nextCls++;
269 // построение отображения алфавитов входных символов.
270
270 // поскольку символ DFAConst.UNCLASSIFIED_INPUT может иметь
271 // сохраняем DFAConst.UNCLASSIFIED_INPUT
271 // специальное значение, тогда сохраним минимальный класс,
272 var cls = item.Contains(DFAConst.UNCLASSIFIED_INPUT) ? DFAConst.UNCLASSIFIED_INPUT : nextCls;
272 // содержащий этот символ на томже месте.
273
273
274 foreach (var a in item)
274 var nextCls = 0;
275 alphabetMap[a] = cls;
275 foreach (var item in minClasses) {
276
276 if (nextCls == DFAConst.UNCLASSIFIED_INPUT)
277 nextCls++;
277 nextCls++;
278 }
278
279
279 // сохраняем DFAConst.UNCLASSIFIED_INPUT
280 // построение автомата
280 var cls = item.Contains(DFAConst.UNCLASSIFIED_INPUT) ? DFAConst.UNCLASSIFIED_INPUT : nextCls;
281 optimalDFA.SetInitialState(stateMap[m_initialState]);
281
282
282 foreach (var a in item)
283 foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
283 alphabetMap[a] = cls;
284 optimalDFA.MarkFinalState(sf);
284
285
285 nextCls++;
286 foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
286 }
287 optimalDFA.Add(t);
287
288 }
288 // построение автомата
289
289 optimalDFA.SetInitialState(stateMap[m_initialState]);
290 protected void PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
290
291 Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
291 foreach (var sf in m_finalStates.Select(s => stateMap[s]).Distinct())
292 Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
292 optimalDFA.MarkFinalState(sf);
293
293
294 foreach(var t in m_transitions)
294 foreach (var t in m_transitions.Select(t => new AutomatonTransition(stateMap[t.s1],stateMap[t.s2],alphabetMap[t.edge])).Distinct())
295 Console.WriteLine(
295 optimalDFA.Add(t);
296 "[{0}] -{{{1}}}-> [{2}]{3}",
296 }
297 String.Join(",", stateAlphabet.GetSymbols(t.s1)),
297
298 String.Join("", inputAlphabet.GetSymbols(t.edge)),
298 protected void PrintDFA<TInput, TState>(IAlphabet<TInput> inputAlphabet, IAlphabet<TState> stateAlphabet) {
299 String.Join(",", stateAlphabet.GetSymbols(t.s2)),
299 Safe.ArgumentNotNull(inputAlphabet, "inputAlphabet");
300 m_finalStates.Contains(t.s2) ? "$" : ""
300 Safe.ArgumentNotNull(stateAlphabet, "stateAlphabet");
301 );
301
302 }
302 foreach(var t in m_transitions)
303
303 Console.WriteLine(
304 }
304 "[{0}] -{{{1}}}-> [{2}]{3}",
305 }
305 String.Join(",", stateAlphabet.GetSymbols(t.s1)),
306 String.Join("", inputAlphabet.GetSymbols(t.edge)),
307 String.Join(",", stateAlphabet.GetSymbols(t.s2)),
308 m_finalStates.Contains(t.s2) ? "$" : ""
309 );
310 }
311
312 }
313 }
@@ -1,94 +1,50
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Diagnostics;
4 using System.Diagnostics;
5 using System.Linq;
5 using System.Linq;
6
6
7 namespace Implab.Automaton {
7 namespace Implab.Automaton {
8 /// <summary>
8 /// <summary>
9 /// Indexed alphabet is the finite set of symbols where each symbol has a zero-based unique index.
9 /// Indexed alphabet is the finite set of symbols where each symbol has a zero-based unique index.
10 /// </summary>
10 /// </summary>
11 /// <remarks>
11 /// <remarks>
12 /// Indexed alphabets are usefull in bulting efficient translations from source alphabet
12 /// Indexed alphabets are usefull in bulting efficient translations from source alphabet
13 /// to the input alphabet of the automaton. It's assumed that the index to the symbol match
13 /// to the input alphabet of the automaton. It's assumed that the index to the symbol match
14 /// is well known and documented.
14 /// is well known and documented.
15 /// </remarks>
15 /// </remarks>
16 public abstract class IndexedAlphabetBase<T> : IAlphabetBuilder<T> {
16 public abstract class IndexedAlphabetBase<T> : MapAlphabet<T> {
17 int m_nextId = 1;
18 readonly int[] m_map;
19
20 protected IndexedAlphabetBase(int mapSize) {
21 m_map = new int[mapSize];
22 }
23
24 protected IndexedAlphabetBase(int[] map) {
25 Debug.Assert(map != null && map.Length > 0);
26 Debug.Assert(map.All(x => x >= 0));
27
28 m_map = map;
29 m_nextId = map.Max() + 1;
30 }
31
32 public int DefineSymbol(T symbol) {
33 var index = GetSymbolIndex(symbol);
34 if (m_map[index] == DFAConst.UNCLASSIFIED_INPUT)
35 m_map[index] = m_nextId++;
36 return m_map[index];
37 }
38
39 public int DefineSymbol(T symbol, int cls) {
40 var index = GetSymbolIndex(symbol);
41 m_map[index] = cls;
42 m_nextId = Math.Max(cls + 1, m_nextId);
43 return cls;
44 }
45
17
46 public int DefineClass(IEnumerable<T> symbols) {
18 protected IndexedAlphabetBase() :base(true, null) {
47 return DefineClass(symbols, m_nextId);
48 }
49
50 public int DefineClass(IEnumerable<T> symbols, int cls) {
51 Safe.ArgumentNotNull(symbols, "symbols");
52 symbols = symbols.Distinct();
53
54 foreach (var symbol in symbols)
55 m_map[GetSymbolIndex(symbol)] = cls;
56
57 m_nextId = Math.Max(cls + 1, m_nextId);
58
59 return cls;
60 }
61
62 public virtual int Translate(T symbol) {
63 return m_map[GetSymbolIndex(symbol)];
64 }
65
66 public int Count {
67 get { return m_nextId; }
68 }
69
70 public bool Contains(T symbol) {
71 return true;
72 }
73
74 public IEnumerable<T> GetSymbols(int cls) {
75 for (var i = 0; i < m_map.Length; i++)
76 if (m_map[i] == cls)
77 yield return GetSymbolByIndex(i);
78 }
19 }
79
20
80 public abstract int GetSymbolIndex(T symbol);
21 public abstract int GetSymbolIndex(T symbol);
81
22
82 public abstract T GetSymbolByIndex(int index);
83
84 public abstract IEnumerable<T> InputSymbols { get; }
85
86 /// <summary>
23 /// <summary>
87 /// Gets the translation map from the index of the symbol to it's class this is usefull for the optimized input symbols transtaion.
24 /// Gets the translation map from the index of the symbol to it's class this is usefull for the optimized input symbols transtaion.
88 /// </summary>
25 /// </summary>
26 /// <remarks>
27 /// The map is continous and start from the symbol with zero code. The last symbol
28 /// in the map is the last classified symbol in the alphabet, i.e. the map can be
29 /// shorter then the whole alphabet.
30 /// </remarks>
89 /// <returns>The translation map.</returns>
31 /// <returns>The translation map.</returns>
90 public int[] GetTranslationMap() {
32 public int[] GetTranslationMap() {
91 return m_map;
33 Dictionary<int,int> map = new Dictionary<int, int>();
34
35 int max;
36 foreach (var p in Mappings) {
37 var index = GetSymbolIndex(p.Key);
38 max = Math.Max(max, index);
39 map[index] = p.Value;
40 }
41
42 var result = new int[max + 1];
43
44 for (int i = 0; i < result.Length; i++)
45 map.TryGetValue(i, out result[i]);
46
47 return result;
92 }
48 }
93 }
49 }
94 }
50 }
@@ -1,77 +1,84
1 using System;
1 using System;
2 using System.Collections.Generic;
2 using System.Collections.Generic;
3 using System.Linq;
3 using System.Linq;
4
4
5 namespace Implab.Automaton {
5 namespace Implab.Automaton {
6 public class MapAlphabet<T> : IAlphabetBuilder<T> {
6 public class MapAlphabet<T> : IAlphabetBuilder<T> {
7 readonly Dictionary<T,int> m_map;
7 readonly Dictionary<T,int> m_map;
8 int m_nextCls;
8 int m_nextCls;
9 readonly bool m_supportUnclassified;
9 readonly bool m_supportUnclassified;
10
10
11 public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
11 public MapAlphabet(bool supportUnclassified, IEqualityComparer<T> comparer) {
12 m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
12 m_map = comparer != null ? new Dictionary<T, int>(comparer) : new Dictionary<T,int>();
13 m_supportUnclassified = supportUnclassified;
13 m_supportUnclassified = supportUnclassified;
14 m_nextCls = supportUnclassified ? 1 : 0;
14 m_nextCls = supportUnclassified ? 1 : 0;
15 }
15 }
16
16
17 #region IAlphabetBuilder implementation
17 #region IAlphabetBuilder implementation
18
18
19 public int DefineSymbol(T symbol) {
19 public int DefineSymbol(T symbol) {
20 int cls;
20 int cls;
21 return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
21 return m_map.TryGetValue(symbol, out cls) ? cls : DefineSymbol(symbol, m_nextCls);
22 }
22 }
23
23
24 public int DefineSymbol(T symbol, int cls) {
24 public int DefineSymbol(T symbol, int cls) {
25 Safe.ArgumentAssert(cls >= 0, "cls");
25 Safe.ArgumentAssert(cls >= 0, "cls");
26
26
27 m_nextCls = Math.Max(cls + 1, m_nextCls);
27 m_nextCls = Math.Max(cls + 1, m_nextCls);
28 m_map.Add(symbol, cls);
28 m_map.Add(symbol, cls);
29 return cls;
29 return cls;
30 }
30 }
31
31
32 public int DefineClass(IEnumerable<T> symbols) {
32 public int DefineClass(IEnumerable<T> symbols) {
33 return DefineClass(symbols, m_nextCls);
33 return DefineClass(symbols, m_nextCls);
34 }
34 }
35
35
36 public int DefineClass(IEnumerable<T> symbols, int cls) {
36 public int DefineClass(IEnumerable<T> symbols, int cls) {
37 Safe.ArgumentAssert(cls >= 0, "cls");
37 Safe.ArgumentAssert(cls >= 0, "cls");
38 Safe.ArgumentNotNull(symbols, "symbols");
38 Safe.ArgumentNotNull(symbols, "symbols");
39
39
40 m_nextCls = Math.Max(cls + 1, m_nextCls);
40 m_nextCls = Math.Max(cls + 1, m_nextCls);
41
41
42 foreach (var symbol in symbols)
42 foreach (var symbol in symbols)
43 m_map[symbol] = cls;
43 m_map[symbol] = cls;
44 return cls;
44 return cls;
45 }
45 }
46
46
47 #endregion
47 #endregion
48
48
49 #region IAlphabet implementation
49 #region IAlphabet implementation
50
50
51 public int Translate(T symbol) {
51 public int Translate(T symbol) {
52 int cls;
52 int cls;
53 if (m_map.TryGetValue(symbol, out cls))
53 if (m_map.TryGetValue(symbol, out cls))
54 return cls;
54 return cls;
55 if (!m_supportUnclassified)
55 if (!m_supportUnclassified)
56 throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
56 throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
57 return DFAConst.UNCLASSIFIED_INPUT;
57 return DFAConst.UNCLASSIFIED_INPUT;
58 }
58 }
59
59
60 public int Count {
60 public int Count {
61 get {
61 get {
62 return m_nextCls;
62 return m_nextCls;
63 }
63 }
64 }
64 }
65
65
66 public bool Contains(T symbol) {
66 public bool Contains(T symbol) {
67 return m_supportUnclassified || m_map.ContainsKey(symbol);
67 return m_supportUnclassified || m_map.ContainsKey(symbol);
68 }
68 }
69
69
70
70
71 public IEnumerable<T> GetSymbols(int cls) {
71 public IEnumerable<T> GetSymbols(int cls) {
72 Safe.ArgumentAssert(cls > 0, "cls");
72 return m_map.Where(p => p.Value == cls).Select(p => p.Key);
73 return m_map.Where(p => p.Value == cls).Select(p => p.Key);
73 }
74 }
74 #endregion
75 #endregion
76
77 public IEnumerable<KeyValuePair<T,int>> Mappings {
78 get {
79 return m_map;
80 }
81 }
75 }
82 }
76 }
83 }
77
84
@@ -1,89 +1,98
1 using Implab;
1 using Implab;
2 using System;
2 using System;
3 using System.Collections.Generic;
3 using System.Collections.Generic;
4 using System.Linq;
4 using System.Linq;
5
5
6 namespace Implab.Automaton.RegularExpressions {
6 namespace Implab.Automaton.RegularExpressions {
7 /// <summary>
7 /// <summary>
8 /// Базовый абстрактный класс. Грамматика, позволяет формулировать выражения над алфавитом типа <c>char</c>.
8 /// Базовый абстрактный класс. Грамматика, позволяет формулировать выражения над алфавитом типа <c>char</c>.
9 /// </summary>
9 /// </summary>
10 public abstract class Grammar<TSymbol, TTag> {
10 public abstract class Grammar<TSymbol, TTag> {
11
11
12 protected abstract IAlphabetBuilder<TSymbol> AlphabetBuilder {
12 protected abstract IAlphabetBuilder<TSymbol> AlphabetBuilder {
13 get;
13 get;
14 }
14 }
15
15
16 protected SymbolToken<TTag> UnclassifiedToken() {
16 protected SymbolToken<TTag> UnclassifiedToken() {
17 return new SymbolToken<TTag>(DFAConst.UNCLASSIFIED_INPUT);
17 return new SymbolToken<TTag>(DFAConst.UNCLASSIFIED_INPUT);
18 }
18 }
19
19
20 protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
20 protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
21 Safe.ArgumentNotNull(alphabet, "alphabet");
21 Safe.ArgumentNotNull(alphabet, "alphabet");
22
22
23 foreach (var ch in alphabet)
23 foreach (var ch in alphabet)
24 AlphabetBuilder.DefineSymbol(ch);
24 AlphabetBuilder.DefineSymbol(ch);
25 }
25 }
26
26
27 protected Token<TTag> SymbolToken(TSymbol symbol) {
27 protected Token<TTag> SymbolToken(TSymbol symbol) {
28 return Token<TTag>.New(TranslateOrAdd(symbol));
28 return Token<TTag>.New(TranslateOrAdd(symbol));
29 }
29 }
30
30
31 protected Token<TTag> SymbolToken(IEnumerable<TSymbol> symbols) {
31 protected Token<TTag> SymbolToken(IEnumerable<TSymbol> symbols) {
32 Safe.ArgumentNotNull(symbols, "symbols");
32 Safe.ArgumentNotNull(symbols, "symbols");
33
33
34 return Token<TTag>.New(TranslateOrAdd(symbols).ToArray());
34 return Token<TTag>.New(TranslateOrAdd(symbols).ToArray());
35 }
35 }
36
36
37 protected Token<TTag> SymbolSetToken(params TSymbol[] set) {
37 protected Token<TTag> SymbolSetToken(params TSymbol[] set) {
38 return SymbolToken(set);
38 return SymbolToken(set);
39 }
39 }
40
40
41 int TranslateOrAdd(TSymbol ch) {
41 int TranslateOrAdd(TSymbol ch) {
42 var t = AlphabetBuilder.Translate(ch);
42 var t = AlphabetBuilder.Translate(ch);
43 if (t == DFAConst.UNCLASSIFIED_INPUT)
43 if (t == DFAConst.UNCLASSIFIED_INPUT)
44 t = AlphabetBuilder.DefineSymbol(ch);
44 t = AlphabetBuilder.DefineSymbol(ch);
45 return t;
45 return t;
46 }
46 }
47
47
48 IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
48 IEnumerable<int> TranslateOrAdd(IEnumerable<TSymbol> symbols) {
49 return symbols.Distinct().Select(TranslateOrAdd);
49 return symbols.Distinct().Select(TranslateOrAdd);
50 }
50 }
51
51
52 int TranslateOrDie(TSymbol ch) {
52 int TranslateOrDie(TSymbol ch) {
53 var t = AlphabetBuilder.Translate(ch);
53 var t = AlphabetBuilder.Translate(ch);
54 if (t == DFAConst.UNCLASSIFIED_INPUT)
54 if (t == DFAConst.UNCLASSIFIED_INPUT)
55 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
55 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
56 return t;
56 return t;
57 }
57 }
58
58
59 IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
59 IEnumerable<int> TranslateOrDie(IEnumerable<TSymbol> symbols) {
60 return symbols.Distinct().Select(TranslateOrDie);
60 return symbols.Distinct().Select(TranslateOrDie);
61 }
61 }
62
62
63 protected Token<TTag> SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
63 protected Token<TTag> SymbolTokenExcept(IEnumerable<TSymbol> symbols) {
64 Safe.ArgumentNotNull(symbols, "symbols");
64 Safe.ArgumentNotNull(symbols, "symbols");
65
65
66 return Token<TTag>.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
66 return Token<TTag>.New( Enumerable.Range(0, AlphabetBuilder.Count).Except(TranslateOrDie(symbols)).ToArray() );
67 }
67 }
68
68
69 protected abstract IAlphabetBuilder<TSymbol> CreateAlphabet();
69 protected abstract IndexedAlphabetBase<TSymbol> CreateAlphabet();
70
70
71 protected RegularDFA<TSymbol, TTag> BuildDFA(Token<TTag> regexp) {
71 protected ScannerContext<TTag> BuildScannerContext(Token<TTag> regexp) {
72
72
73 var dfa = new RegularDFA<TSymbol, TTag>(AlphabetBuilder);
73 var dfa = new RegularDFA<TSymbol, TTag>(AlphabetBuilder);
74
74
75 var visitor = new RegularExpressionVisitor<TTag>();
75 var visitor = new RegularExpressionVisitor<TTag>();
76 regexp.Accept( visitor );
76 regexp.Accept( visitor );
77
77
78 visitor.BuildDFA(dfa);
78 visitor.BuildDFA(dfa);
79
79
80 if (dfa.IsFinalState(dfa.InitialState))
80 if (dfa.IsFinalState(dfa.InitialState))
81 throw new ApplicationException("The specified language contains empty token");
81 throw new ApplicationException("The specified language contains empty token");
82
82
83 return dfa.Optimize(CreateAlphabet());
83 var ab = CreateAlphabet();
84 var optimal = dfa.Optimize(ab);
85
86 return new ScannerContext<TTag>(
87 optimal.CreateTransitionTable(),
88 optimal.CreateFinalStateTable(),
89 optimal.CreateTagTable(),
90 optimal.InitialState,
91 ab.GetTranslationMap()
92 );
84 }
93 }
85
94
86 }
95 }
87
96
88
97
89 }
98 }
@@ -1,89 +1,84
1 using System;
1 using System;
2 using System.Collections.Generic;
2 using System.Collections.Generic;
3 using System.Linq;
3 using System.Linq;
4
4
5 namespace Implab.Automaton.RegularExpressions {
5 namespace Implab.Automaton.RegularExpressions {
6 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
6 public class RegularDFA<TInput, TTag> : DFATable, ITaggedDFABuilder<TTag> {
7
7
8 readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
8 readonly Dictionary<int,TTag[]> m_tags = new Dictionary<int, TTag[]>();
9 readonly IAlphabet<TInput> m_alphabet;
9 readonly IAlphabet<TInput> m_alphabet;
10
10
11 public RegularDFA(IAlphabet<TInput> alphabet) {
11 public RegularDFA(IAlphabet<TInput> alphabet) {
12 Safe.ArgumentNotNull(alphabet, "aplhabet");
12 Safe.ArgumentNotNull(alphabet, "aplhabet");
13
13
14 m_alphabet = alphabet;
14 m_alphabet = alphabet;
15 }
15 }
16
16
17
17
18 public IAlphabet<TInput> InputAlphabet {
18 public IAlphabet<TInput> InputAlphabet {
19 get {
19 get {
20 return m_alphabet;
20 return m_alphabet;
21 }
21 }
22 }
22 }
23
23
24 public void MarkFinalState(int s, TTag[] tags) {
24 public void MarkFinalState(int s, TTag[] tags) {
25 MarkFinalState(s);
25 MarkFinalState(s);
26 SetStateTag(s, tags);
26 SetStateTag(s, tags);
27 }
27 }
28
28
29 public void SetStateTag(int s, TTag[] tags) {
29 public void SetStateTag(int s, TTag[] tags) {
30 Safe.ArgumentNotNull(tags, "tags");
30 Safe.ArgumentNotNull(tags, "tags");
31 m_tags[s] = tags;
31 m_tags[s] = tags;
32 }
32 }
33
33
34 public TTag[] GetStateTag(int s) {
34 public TTag[] GetStateTag(int s) {
35 TTag[] tags;
35 TTag[] tags;
36 return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
36 return m_tags.TryGetValue(s, out tags) ? tags : new TTag[0];
37 }
37 }
38
38
39 public new DFAStateDescriptor<TTag>[] CreateTransitionTable() {
39 public TTag[][] CreateTagTable() {
40 var table = new DFAStateDescriptor<TTag>[StateCount];
40 var table = new TTag[StateCount][];
41
41
42 foreach (var t in this) {
42 foreach (var pair in m_tags)
43 if (table[t.s1].transitions == null)
43 table[pair.Key] = pair.Value;
44 table[t.s1] = new DFAStateDescriptor<TTag>(AlphabetSize, IsFinalState(t.s1), GetStateTag(t.s1));
45 if (table[t.s2].transitions == null)
46 table[t.s2] = new DFAStateDescriptor<TTag>(AlphabetSize, IsFinalState(t.s2), GetStateTag(t.s2));
47 table[t.s1].transitions[t.edge] = t.s2;
48 }
49
44
50 return table;
45 return table;
51 }
46 }
52
47
53 /// <summary>
48 /// <summary>
54 /// Optimize the specified alphabet.
49 /// Optimize the specified alphabet.
55 /// </summary>
50 /// </summary>
56 /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
51 /// <param name="alphabet">Пустой алфавит, который будет зполнен в процессе оптимизации.</param>
57 public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
52 public RegularDFA<TInput,TTag> Optimize(IAlphabetBuilder<TInput> alphabet) {
58 Safe.ArgumentNotNull(alphabet, "alphabet");
53 Safe.ArgumentNotNull(alphabet, "alphabet");
59
54
60 var dfa = new RegularDFA<TInput, TTag>(alphabet);
55 var dfa = new RegularDFA<TInput, TTag>(alphabet);
61
56
62 var states = new DummyAlphabet(StateCount);
57 var states = new DummyAlphabet(StateCount);
63 var alphaMap = new Dictionary<int,int>();
58 var alphaMap = new Dictionary<int,int>();
64 var stateMap = new Dictionary<int,int>();
59 var stateMap = new Dictionary<int,int>();
65
60
66 Optimize(dfa, alphaMap, stateMap);
61 Optimize(dfa, alphaMap, stateMap);
67
62
68 // mark tags in the new DFA
63 // mark tags in the new DFA
69 foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
64 foreach (var g in m_tags.Where(x => x.Key < StateCount).GroupBy(x => stateMap[x.Key], x => x.Value ))
70 dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
65 dfa.SetStateTag(g.Key, g.SelectMany(x => x).ToArray());
71
66
72 // make the alphabet for the new DFA
67 // make the alphabet for the new DFA
73 foreach (var pair in alphaMap)
68 foreach (var pair in alphaMap)
74 alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
69 alphabet.DefineClass(m_alphabet.GetSymbols(pair.Key), pair.Value);
75
70
76 return dfa;
71 return dfa;
77 }
72 }
78
73
79 protected override IEnumerable<HashSet<int>> GroupFinalStates() {
74 protected override IEnumerable<HashSet<int>> GroupFinalStates() {
80 var arrayComparer = new CustomEqualityComparer<TTag[]>(
75 var arrayComparer = new CustomEqualityComparer<TTag[]>(
81 (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
76 (x,y) => x.Length == y.Length && x.All(it => y.Contains(it)),
82 x => x.Sum(it => x.GetHashCode())
77 x => x.Sum(it => x.GetHashCode())
83 );
78 );
84 return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
79 return FinalStates.GroupBy(x => m_tags[x], arrayComparer).Select(g => new HashSet<int>(g));
85 }
80 }
86
81
87 }
82 }
88 }
83 }
89
84
@@ -1,29 +1,25
1 using System.Collections.Generic;
1 using System.Collections.Generic;
2 using System.Linq;
2 using System.Linq;
3 using Implab.Automaton;
3 using Implab.Automaton;
4
4
5 namespace Implab.Formats {
5 namespace Implab.Formats {
6 public class ByteAlphabet : IndexedAlphabetBase<byte> {
6 public class ByteAlphabet : IndexedAlphabetBase<byte> {
7 public ByteAlphabet() : base(byte.MaxValue + 1){
7 public ByteAlphabet() {
8 }
8 }
9
9
10 #region implemented abstract members of IndexedAlphabetBase
10 #region implemented abstract members of IndexedAlphabetBase
11
11
12 public override int GetSymbolIndex(byte symbol) {
12 public override int GetSymbolIndex(byte symbol) {
13 return (int)symbol;
13 return (int)symbol;
14 }
14 }
15
15
16 public override byte GetSymbolByIndex(int index) {
17 return (byte)index;
18 }
19
20 public IEnumerable<byte> InputSymbols {
16 public IEnumerable<byte> InputSymbols {
21 get {
17 get {
22 return Enumerable.Range(byte.MinValue, byte.MaxValue).Cast<byte>();
18 return Enumerable.Range(byte.MinValue, byte.MaxValue).Cast<byte>();
23 }
19 }
24 }
20 }
25
21
26 #endregion
22 #endregion
27 }
23 }
28 }
24 }
29
25
@@ -1,24 +1,19
1 using System.Collections.Generic;
1 using System.Collections.Generic;
2 using System.Linq;
2 using System.Linq;
3 using Implab.Automaton;
3 using Implab.Automaton;
4
4
5 namespace Implab.Formats {
5 namespace Implab.Formats {
6 public class CharAlphabet: IndexedAlphabetBase<char> {
6 public class CharAlphabet: IndexedAlphabetBase<char> {
7
7
8 public CharAlphabet()
8 public CharAlphabet() {
9 : base(char.MaxValue + 1) {
10 }
9 }
11
10
12 public override int GetSymbolIndex(char symbol) {
11 public override int GetSymbolIndex(char symbol) {
13 return symbol;
12 return symbol;
14 }
13 }
15
14
16 public override char GetSymbolByIndex(int index) {
15 public IEnumerable<char> InputSymbols {
17 return (char)index;
18 }
19
20 public override IEnumerable<char> InputSymbols {
21 get { return Enumerable.Range(char.MinValue, char.MaxValue).Cast<char>(); }
16 get { return Enumerable.Range(char.MinValue, char.MaxValue).Cast<char>(); }
22 }
17 }
23 }
18 }
24 }
19 }
@@ -1,116 +1,109
1 using System.Linq;
1 using System.Linq;
2 using Implab.Automaton.RegularExpressions;
2 using Implab.Automaton.RegularExpressions;
3 using System;
3 using System;
4 using Implab.Automaton;
4 using Implab.Automaton;
5
5
6 namespace Implab.Formats.JSON {
6 namespace Implab.Formats.JSON {
7 class JSONGrammar : Grammar<char,JSONGrammar.TokenType> {
7 class JSONGrammar : Grammar<char,JSONGrammar.TokenType> {
8 public enum TokenType {
8 public enum TokenType {
9 None,
9 None,
10 BeginObject,
10 BeginObject,
11 EndObject,
11 EndObject,
12 BeginArray,
12 BeginArray,
13 EndArray,
13 EndArray,
14 String,
14 String,
15 Number,
15 Number,
16 Literal,
16 Literal,
17 NameSeparator,
17 NameSeparator,
18 ValueSeparator,
18 ValueSeparator,
19
19
20 StringBound,
20 StringBound,
21 EscapedChar,
21 EscapedChar,
22 UnescapedChar,
22 UnescapedChar,
23 EscapedUnicode,
23 EscapedUnicode
24
25 Minus,
26 Plus,
27 Sign,
28 Integer,
29 Dot,
30 Exp
31 }
24 }
32
25
33 static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();
26 static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();
34
27
35 public static JSONGrammar Instance {
28 public static JSONGrammar Instance {
36 get { return _instance.Value; }
29 get { return _instance.Value; }
37 }
30 }
38
31
39 readonly RegularDFA<char, TokenType> m_jsonDFA;
32 readonly ScannerContext<TokenType> m_jsonDFA;
40 readonly RegularDFA<char, TokenType> m_stringDFA;
33 readonly ScannerContext<TokenType> m_stringDFA;
41
34
42 public JSONGrammar() {
35 public JSONGrammar() {
43 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
36 DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
44 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
37 var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
45 var digit9 = SymbolRangeToken('1', '9');
38 var digit9 = SymbolRangeToken('1', '9');
46 var zero = SymbolToken('0');
39 var zero = SymbolToken('0');
47 var digit = zero.Or(digit9);
40 var digit = zero.Or(digit9);
48 var dot = SymbolToken('.');
41 var dot = SymbolToken('.');
49 var minus = SymbolToken('-');
42 var minus = SymbolToken('-');
50 var sign = SymbolSetToken('-', '+');
43 var sign = SymbolSetToken('-', '+');
51 var expSign = SymbolSetToken('e', 'E');
44 var expSign = SymbolSetToken('e', 'E');
52 var letters = SymbolRangeToken('a', 'z');
45 var letters = SymbolRangeToken('a', 'z');
53 var integer = zero.Or(digit9.Cat(digit.EClosure()));
46 var integer = zero.Or(digit9.Cat(digit.EClosure()));
54 var frac = dot.Cat(digit.Closure());
47 var frac = dot.Cat(digit.Closure());
55 var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
48 var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
56 var quote = SymbolToken('"');
49 var quote = SymbolToken('"');
57 var backSlash = SymbolToken('\\');
50 var backSlash = SymbolToken('\\');
58 var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
51 var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
59 var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
52 var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
60 var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
53 var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
61 var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
54 var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
62 var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
55 var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
63 var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
56 var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
64 var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
57 var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
65 var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
58 var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
66 var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
59 var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);
67
60
68 var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
61 var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
69 var literal = letters.Closure();
62 var literal = letters.Closure();
70 var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
63 var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));
71
64
72 var jsonExpression =
65 var jsonExpression =
73 number.Tag(TokenType.Number)
66 number.Tag(TokenType.Number)
74 .Or(literal.Tag(TokenType.Literal))
67 .Or(literal.Tag(TokenType.Literal))
75 .Or(quote.Tag(TokenType.StringBound))
68 .Or(quote.Tag(TokenType.StringBound))
76 .Or(beginObject.Tag(TokenType.BeginObject))
69 .Or(beginObject.Tag(TokenType.BeginObject))
77 .Or(endObject.Tag(TokenType.EndObject))
70 .Or(endObject.Tag(TokenType.EndObject))
78 .Or(beginArray.Tag(TokenType.BeginArray))
71 .Or(beginArray.Tag(TokenType.BeginArray))
79 .Or(endArray.Tag(TokenType.EndArray))
72 .Or(endArray.Tag(TokenType.EndArray))
80 .Or(nameSep.Tag(TokenType.NameSeparator))
73 .Or(nameSep.Tag(TokenType.NameSeparator))
81 .Or(valueSep.Tag(TokenType.ValueSeparator));
74 .Or(valueSep.Tag(TokenType.ValueSeparator));
82
75
83
76
84 var jsonStringExpression =
77 var jsonStringExpression =
85 quote.Tag(TokenType.StringBound)
78 quote.Tag(TokenType.StringBound)
86 .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
79 .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
87 .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
80 .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
88 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
81 .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
89
82
90
83
91 m_jsonDFA = BuildDFA(jsonExpression);
84 m_jsonDFA = BuildScannerContext(jsonExpression);
92 m_stringDFA = BuildDFA(jsonStringExpression);
85 m_stringDFA = BuildScannerContext(jsonStringExpression);
93 }
86 }
94
87
95 public RegularDFA<char, TokenType> JsonDFA {
88 public ScannerContext<TokenType> JsonDFA {
96 get {
89 get {
97 return m_jsonDFA;
90 return m_jsonDFA;
98 }
91 }
99 }
92 }
100
93
101 public RegularDFA<char,TokenType> JsonStringDFA {
94 public ScannerContext<TokenType> JsonStringDFA {
102 get {
95 get {
103 return m_stringDFA;
96 return m_stringDFA;
104 }
97 }
105 }
98 }
106
99
107 Token<TokenType> SymbolRangeToken(char start, char stop) {
100 Token<TokenType> SymbolRangeToken(char start, char stop) {
108 return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
101 return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
109 }
102 }
110
103
111 protected override IAlphabetBuilder<char> CreateAlphabet() {
104 protected override IAlphabetBuilder<char> CreateAlphabet() {
112 return new CharAlphabet();
105 return new CharAlphabet();
113 }
106 }
114
107
115 }
108 }
116 }
109 }
@@ -1,96 +1,101
1 using System;
1 using System;
2 using System.Globalization;
2 using System.Globalization;
3 using Implab.Automaton;
3 using Implab.Automaton;
4 using System.Text;
5 using Implab.Components;
6 using System.IO;
7 using Implab.Automaton.RegularExpressions;
4
8
5 namespace Implab.Formats.JSON {
9 namespace Implab.Formats.JSON {
6 /// <summary>
10 /// <summary>
7 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
11 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
8 /// </summary>
12 /// </summary>
9 public class JSONScanner : Scanner<object> {
13 public class JSONScanner : Disposable {
10 char[] m_stringBuffer;
14 readonly StringBuilder m_builder = new StringBuilder();
11 DFAStateDescriptior<>[] m_stringDFA;
15
12 int[] m_stringAlphabet;
16 readonly ScannerContext<JSONGrammar.TokenType> m_jsonScanner = JSONGrammar.Instance.JsonDFA;
17 readonly ScannerContext<JSONGrammar.TokenType> m_stringScanner = JSONGrammar.Instance.JsonStringDFA;
18
19
20 readonly TextScanner m_scanner;
13
21
14 /// <summary>
22 /// <summary>
15 /// Создает новый экземпляр сканнера
23 /// Создает новый экземпляр сканнера
16 /// </summary>
24 /// </summary>
17 public JSONScanner()
25 public JSONScanner(string text) {
18 : base(JSONGrammar.Instance.JsonDFA.GetTransitionTable(), JSONGrammar.Instance.JsonDFA.Alphabet.GetTranslationMap()) {
26 Safe.ArgumentNotEmpty(text, "text");
19 m_stringBuffer = new char[1024];
27
20 var dfa = JSONGrammar.Instance.JsonStringDFA;
28 m_scanner = new StringScanner(text);
21 m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
29 }
22 m_stringDFA = dfa.States;
30
31 public JSONScanner(TextReader reader, int bufferMax, int chunkSize) {
32 Safe.ArgumentNotNull(reader, "reader");
33
34 m_scanner = new ReaderScanner(reader);
23 }
35 }
24
36
25 /// <summary>
37 /// <summary>
26 /// Читает следующий лексический элемент из входных данных.
38 /// Читает следующий лексический элемент из входных данных.
27 /// </summary>
39 /// </summary>
28 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
40 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
29 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
41 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
30 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
42 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
31 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
43 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
32 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
44 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
33 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
45 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
34 if (ReadTokenInternal()) {
46 JSONGrammar.TokenType[] tag;
35 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
47 if (m_jsonScanner.Execute(m_scanner, out tag)) {
48 switch (tag[0]) {
36 case JSONGrammar.TokenType.StringBound:
49 case JSONGrammar.TokenType.StringBound:
37 tokenValue = ReadString();
50 tokenValue = ReadString();
38 tokenType = JsonTokenType.String;
51 tokenType = JsonTokenType.String;
39 break;
52 break;
40 case JSONGrammar.TokenType.Number:
53 case JSONGrammar.TokenType.Number:
41 tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
54 tokenValue = Double.Parse(m_scanner.GetTokenValue(), CultureInfo.InvariantCulture);
42 tokenType = JsonTokenType.Number;
55 tokenType = JsonTokenType.Number;
43 break;
56 break;
44 default:
57 default:
45 tokenType = (JsonTokenType)m_currentState.tag[0];
58 tokenType = (JsonTokenType)tag[0];
46 tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
59 tokenValue = m_scanner.GetTokenValue();
47 break;
60 break;
48 }
61 }
49 return true;
62 return true;
50 }
63 }
51 tokenValue = null;
64 tokenValue = null;
52 tokenType = JsonTokenType.None;
65 tokenType = JsonTokenType.None;
53 return false;
66 return false;
54 }
67 }
55
68
56 string ReadString() {
69 string ReadString() {
57 int pos = 0;
70 int pos = 0;
58 Switch(m_stringDFA, m_stringAlphabet);
71 char[] buf = new char[6]; // the buffer for unescaping chars
59 while (ReadTokenInternal()) {
72
60 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
73 JSONGrammar.TokenType[] tag;
74 m_builder.Clear();
75
76 while (m_stringScanner.Execute(m_scanner, out tag)) {
77 switch (tag[0]) {
61 case JSONGrammar.TokenType.StringBound:
78 case JSONGrammar.TokenType.StringBound:
62 Restore();
79 return m_builder.ToString();
63 return new String(m_stringBuffer, 0, pos);
64 case JSONGrammar.TokenType.UnescapedChar:
80 case JSONGrammar.TokenType.UnescapedChar:
65 EnsureStringBufferSize(pos + m_tokenLen);
81 m_scanner.CopyTokenTo(m_builder);
66 Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
67 pos += m_tokenLen;
68 break;
82 break;
69 case JSONGrammar.TokenType.EscapedUnicode:
83 case JSONGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
70 EnsureStringBufferSize(pos + 1);
84 m_scanner.CopyTokenTo(buf, 0);
71 m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
85 m_builder.Append(StringTranslator.TranslateHexUnicode(buf, 2));
72 pos++;
86 pos++;
73 break;
87 break;
74 case JSONGrammar.TokenType.EscapedChar:
88 case JSONGrammar.TokenType.EscapedChar: // \t - escape sequence
75 EnsureStringBufferSize(pos + 1);
89 m_scanner.CopyTokenTo(buf, 0);
76 m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
90 m_builder.Append(StringTranslator.TranslateEscapedChar(buf[1]));
77 pos++;
78 break;
91 break;
79 default:
92 default:
80 break;
93 break;
81 }
94 }
82
95
83 }
96 }
84
97
85 throw new ParserException("Unexpected end of data");
98 throw new ParserException("Unexpected end of data");
86 }
99 }
87
88 void EnsureStringBufferSize(int size) {
89 if (size > m_stringBuffer.Length) {
90 var newBuffer = new char[size];
91 m_stringBuffer.CopyTo(newBuffer, 0);
92 m_stringBuffer = newBuffer;
93 }
94 }
95 }
100 }
96 }
101 }
@@ -1,96 +1,95
1 using Implab;
1 using Implab;
2 using Implab.Parsing;
2 using Implab.Formats;
3 using System;
3 using System;
4 using System.Collections.Generic;
4 using System.Collections.Generic;
5 using System.Diagnostics;
5 using System.Diagnostics;
6 using System.Linq;
6 using System.Linq;
7 using System.Text;
7 using System.Text;
8 using System.Threading.Tasks;
8 using System.Threading.Tasks;
9
9
10 namespace Implab.JSON {
10 namespace Implab.Formats.JSON {
11 /// <summary>
11 /// <summary>
12 /// Класс для преобразования экранированной строки JSON
12 /// Класс для преобразования экранированной строки JSON
13 /// </summary>
13 /// </summary>
14 public class StringTranslator : Scanner {
14 public class StringTranslator : TextScanner<JSONGrammar.TokenType> {
15 static readonly char[] _escMap;
15 static readonly char[] _escMap;
16 static readonly int[] _hexMap;
16 static readonly int[] _hexMap;
17
17
18 static StringTranslator() {
18 static StringTranslator() {
19 var chars = new char[] { 'b', 'f', 't', 'r', 'n', '\\', '/' };
19 var chars = new char[] { 'b', 'f', 't', 'r', 'n', '\\', '/' };
20 var vals = new char[] { '\b', '\f', '\t', '\r', '\n', '\\', '/' };
20 var vals = new char[] { '\b', '\f', '\t', '\r', '\n', '\\', '/' };
21
21
22 _escMap = new char[chars.Max() + 1];
22 _escMap = new char[chars.Max() + 1];
23
23
24 for (int i = 0; i < chars.Length; i++)
24 for (int i = 0; i < chars.Length; i++)
25 _escMap[chars[i]] = vals[i];
25 _escMap[chars[i]] = vals[i];
26
26
27 var hexs = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' };
27 var hexs = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' };
28 var ints = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 10, 11, 12, 13, 14, 15 };
28 var ints = new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 10, 11, 12, 13, 14, 15 };
29
29
30 _hexMap = new int[hexs.Max() + 1];
30 _hexMap = new int[hexs.Max() + 1];
31
31
32 for (int i = 0; i < hexs.Length; i++)
32 for (int i = 0; i < hexs.Length; i++)
33 _hexMap[hexs[i]] = ints[i];
33 _hexMap[hexs[i]] = ints[i];
34
34
35 }
35 }
36
36
37 public StringTranslator()
37 public StringTranslator() {
38 : base(JSONGrammar.Instance.JsonStringDFA.States, JSONGrammar.Instance.JsonStringDFA.Alphabet.GetTranslationMap()) {
39 }
38 }
40
39
41 public string Translate(string data) {
40 public string Translate(string data) {
42 Safe.ArgumentNotNull(data, "data");
41 Safe.ArgumentNotNull(data, "data");
43 return Translate(data.ToCharArray());
42 return Translate(data.ToCharArray());
44 }
43 }
45
44
46 public string Translate(char[] data) {
45 public string Translate(char[] data) {
47 Safe.ArgumentNotNull(data, "data");
46 Safe.ArgumentNotNull(data, "data");
48 return Translate(data, data.Length);
47 return Translate(data, data.Length);
49 }
48 }
50
49
51 public string Translate(char[] data, int length) {
50 public string Translate(char[] data, int length) {
52 Safe.ArgumentNotNull(data, "data");
51 Safe.ArgumentNotNull(data, "data");
53 Safe.ArgumentInRange(length, 0, data.Length, "length");
52 Safe.ArgumentInRange(length, 0, data.Length, "length");
54
53
55 var translated = new char[length];
54 var translated = new char[length];
56
55
57 Feed(data,length);
56 Feed(data,length);
58
57
59 int pos = 0;
58 int pos = 0;
60
59
61 while (ReadTokenInternal()) {
60 while (ReadTokenInternal()) {
62 switch ((JSONGrammar.TokenType)TokenTags[0]) {
61 switch ((JSONGrammar.TokenType)Tags[0]) {
63 case JSONGrammar.TokenType.UnescapedChar:
62 case JSONGrammar.TokenType.UnescapedChar:
64 Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
63 Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
65 pos += m_tokenLen;
64 pos += m_tokenLen;
66 break;
65 break;
67 case JSONGrammar.TokenType.EscapedChar:
66 case JSONGrammar.TokenType.EscapedChar:
68 translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
67 translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
69 pos++;
68 pos++;
70 break;
69 break;
71 case JSONGrammar.TokenType.EscapedUnicode:
70 case JSONGrammar.TokenType.EscapedUnicode:
72 translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
71 translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
73 pos++;
72 pos++;
74 break;
73 break;
75 }
74 }
76 }
75 }
77
76
78 return new String(translated, 0, pos);
77 return new String(translated, 0, pos);
79 }
78 }
80
79
81 internal static char TranslateEscapedChar(char symbol) {
80 internal static char TranslateEscapedChar(char symbol) {
82 return _escMap[symbol];
81 return _escMap[symbol];
83 }
82 }
84
83
85 internal static char TranslateHexUnicode(char[] symbols, int offset) {
84 internal static char TranslateHexUnicode(char[] symbols, int offset) {
86 Debug.Assert(symbols != null);
85 Debug.Assert(symbols != null);
87 Debug.Assert(symbols.Length - offset >= 4);
86 Debug.Assert(symbols.Length - offset >= 4);
88
87
89 int value = (_hexMap[symbols[offset]] << 12)
88 int value = (_hexMap[symbols[offset]] << 12)
90 | (_hexMap[symbols[offset + 1]] << 8)
89 | (_hexMap[symbols[offset + 1]] << 8)
91 | (_hexMap[symbols[offset + 2]] << 4)
90 | (_hexMap[symbols[offset + 2]] << 4)
92 | (_hexMap[symbols[offset + 3]]);
91 | (_hexMap[symbols[offset + 3]]);
93 return (char)value;
92 return (char)value;
94 }
93 }
95 }
94 }
96 }
95 }
@@ -1,53 +1,149
1 using System;
1 using System;
2 using Implab.Components;
2 using Implab.Components;
3 using Implab.Automaton.RegularExpressions;
3 using Implab.Automaton.RegularExpressions;
4 using System.Diagnostics;
4 using System.Diagnostics;
5 using Implab.Automaton;
5 using Implab.Automaton;
6 using System.IO;
7 using System.Text;
6
8
7 namespace Implab.Formats {
9 namespace Implab.Formats {
8 public abstract class TextScanner<TTag> : Disposable {
10 public abstract class TextScanner : Disposable {
11 readonly int m_bufferMax;
12 readonly int m_chunkSize;
9
13
10 int m_maxSymbol;
14 char[] m_buffer;
11 int[] m_symbolMap;
12
13 readonly char[] m_buffer;
14 int m_bufferOffset;
15 int m_bufferOffset;
15 int m_bufferSize;
16 int m_bufferSize;
17 int m_tokenOffset;
16 int m_tokenLength;
18 int m_tokenLength;
17
19
18 TTag[] m_tags;
20 /// <summary>
21 /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner`1"/> class.
22 /// </summary>
23 /// <param name="bufferMax">Buffer max.</param>
24 /// <param name="chunkSize">Chunk size.</param>
25 protected TextScanner(int bufferMax, int chunkSize) {
26 Debug.Assert(m_chunkSize <= m_bufferMax);
27
28 m_bufferMax = bufferMax;
29 m_chunkSize = chunkSize;
30 }
19
31
20 protected bool ReadTokenInternal(DFAStateDescriptor<TTag>[] dfa, int state) {
32 /// <summary>
21 Debug.Assert(dfa != null);
33 /// Initializes a new instance of the <see cref="Implab.Formats.TextScanner`1"/> class.
34 /// </summary>
35 /// <param name="buffer">Buffer.</param>
36 protected TextScanner(char[] buffer) {
37 if (buffer != null) {
38 m_buffer = buffer;
39 m_bufferSize = buffer.Length;
40 }
41 }
42
43 /// <summary>
44 /// (hungry) Reads the next token.
45 /// </summary>
46 /// <returns><c>true</c>, if token internal was read, <c>false</c> if there is no more tokens in the stream.</returns>
47 /// <param name="dfa">The transition map for the automaton</param>
48 /// <param name="final">Final states of the automaton.</param>
49 /// <param name="tags">Tags.</param>
50 /// <param name="state">The initial state for the automaton.</param>
51 internal bool ReadToken<TTag>(int[,] dfa, int[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
52 Safe.ArgumentNotNull();
53 m_tokenLength = 0;
54
55 var maxSymbol = alphabet.Length - 1;
22
56
23 do {
57 do {
24 for (var pos = m_bufferOffset; pos < m_bufferSize; pos++) {
58 // after the next chunk is read the offset in the buffer may change
59 int pos = m_bufferOffset + m_tokenLength;
60
61 while(pos < m_bufferSize) {
25 var ch = m_buffer[pos];
62 var ch = m_buffer[pos];
26 state = dfa[state].transitions[m_symbolMap[ch > m_maxSymbol ? m_maxSymbol : ch]];
63
64 state = dfa[state,ch > maxSymbol ? DFAConst.UNCLASSIFIED_INPUT : alphabet[ch]];
27 if (state == DFAConst.UNREACHABLE_STATE)
65 if (state == DFAConst.UNREACHABLE_STATE)
28 break;
66 break;
67
68 pos++;
29 }
69 }
30 } while (Feed());
70
71 m_tokenLength = pos - m_bufferOffset;
72 } while (state != DFAConst.UNREACHABLE_STATE && Feed());
73
74 m_tokenOffset = m_bufferOffset;
75 m_bufferOffset += m_tokenLength;
31
76
32 if (dfa[state].final) {
77 if (final[state]) {
78 tag = tags[state];
79 return true;
80 } else {
81 if (m_bufferOffset == m_bufferSize) {
82 if (m_tokenLength == 0) //EOF
83 return false;
84
85 throw new ParserException();
86 }
87 throw new ParserException(String.Format("Unexpected symbol '{0}'", m_buffer[m_bufferOffset]));
88
89 }
90 }
33
91
34 }
92 protected void Feed(char[] buffer, int offset, int length) {
35
93 m_buffer = buffer;
94 m_bufferOffset = offset;
95 m_bufferSize = offset + length;
36 }
96 }
37
97
38 bool Feed() {
98 protected bool Feed() {
99 if (m_chunkSize <= 0)
100 return false;
101
102 if (m_buffer != null) {
103 var free = m_buffer.Length - m_bufferSize;
104
105 if (free < m_chunkSize) {
106 free += m_chunkSize;
107 var used = m_bufferSize - m_bufferOffset;
108 var size = used + free;
109
110 if (size > m_bufferMax)
111 throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached"), m_bufferMax/1024);
112
113 var temp = new char[size];
39
114
115 var read = Read(temp, used, m_chunkSize);
116 if (read == 0)
117 return false;
118
119 Array.Copy(m_buffer, m_bufferOffset, temp, 0, used);
120
121 m_bufferOffset = 0;
122 m_bufferSize = used + read;
123 m_buffer = temp;
124 }
125 } else {
126 Debug.Assert(m_bufferOffset == 0);
127 m_buffer = new char[m_chunkSize];
128 m_bufferSize = Read(m_buffer, 0, m_chunkSize);
129 return (m_bufferSize != 0);
130 }
40 }
131 }
41
132
42 protected abstract int Read(char[] buffer, int offset, int size);
133 protected abstract int Read(char[] buffer, int offset, int size);
43
134
44 protected TTag[] Tags {
135 public string GetTokenValue() {
45 get {
136 return new String(m_buffer, m_tokenOffset, m_tokenLength);
46 return m_tags;
47 }
48 }
137 }
49
138
139 public void CopyTokenTo(char[] buffer, int offset) {
140 m_buffer.CopyTo(buffer, offset);
141 }
142
143 public void CopyTokenTo(StringBuilder sb) {
144 sb.Append(m_buffer, m_tokenOffset, m_tokenLength);
145 }
50
146
51 }
147 }
52 }
148 }
53
149
@@ -1,274 +1,273
1 <?xml version="1.0" encoding="utf-8"?>
1 <?xml version="1.0" encoding="utf-8"?>
2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 <PropertyGroup>
3 <PropertyGroup>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
4 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
5 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
6 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
7 <OutputType>Library</OutputType>
7 <OutputType>Library</OutputType>
8 <RootNamespace>Implab</RootNamespace>
8 <RootNamespace>Implab</RootNamespace>
9 <AssemblyName>Implab</AssemblyName>
9 <AssemblyName>Implab</AssemblyName>
10 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
10 <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
11 <ReleaseVersion>0.2</ReleaseVersion>
11 <ReleaseVersion>0.2</ReleaseVersion>
12 <ProductVersion>8.0.30703</ProductVersion>
12 <ProductVersion>8.0.30703</ProductVersion>
13 <SchemaVersion>2.0</SchemaVersion>
13 <SchemaVersion>2.0</SchemaVersion>
14 </PropertyGroup>
14 </PropertyGroup>
15 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
15 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16 <DebugSymbols>true</DebugSymbols>
16 <DebugSymbols>true</DebugSymbols>
17 <DebugType>full</DebugType>
17 <DebugType>full</DebugType>
18 <Optimize>false</Optimize>
18 <Optimize>false</Optimize>
19 <OutputPath>bin\Debug</OutputPath>
19 <OutputPath>bin\Debug</OutputPath>
20 <DefineConstants>TRACE;DEBUG;</DefineConstants>
20 <DefineConstants>TRACE;DEBUG;</DefineConstants>
21 <ErrorReport>prompt</ErrorReport>
21 <ErrorReport>prompt</ErrorReport>
22 <WarningLevel>4</WarningLevel>
22 <WarningLevel>4</WarningLevel>
23 <ConsolePause>false</ConsolePause>
23 <ConsolePause>false</ConsolePause>
24 <RunCodeAnalysis>true</RunCodeAnalysis>
24 <RunCodeAnalysis>true</RunCodeAnalysis>
25 </PropertyGroup>
25 </PropertyGroup>
26 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
26 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
27 <DebugType>full</DebugType>
27 <DebugType>full</DebugType>
28 <Optimize>true</Optimize>
28 <Optimize>true</Optimize>
29 <OutputPath>bin\Release</OutputPath>
29 <OutputPath>bin\Release</OutputPath>
30 <ErrorReport>prompt</ErrorReport>
30 <ErrorReport>prompt</ErrorReport>
31 <WarningLevel>4</WarningLevel>
31 <WarningLevel>4</WarningLevel>
32 <ConsolePause>false</ConsolePause>
32 <ConsolePause>false</ConsolePause>
33 </PropertyGroup>
33 </PropertyGroup>
34 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug 4.5|AnyCPU' ">
34 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug 4.5|AnyCPU' ">
35 <DebugSymbols>true</DebugSymbols>
35 <DebugSymbols>true</DebugSymbols>
36 <DebugType>full</DebugType>
36 <DebugType>full</DebugType>
37 <Optimize>false</Optimize>
37 <Optimize>false</Optimize>
38 <OutputPath>bin\Debug</OutputPath>
38 <OutputPath>bin\Debug</OutputPath>
39 <DefineConstants>TRACE;DEBUG;NET_4_5</DefineConstants>
39 <DefineConstants>TRACE;DEBUG;NET_4_5</DefineConstants>
40 <ErrorReport>prompt</ErrorReport>
40 <ErrorReport>prompt</ErrorReport>
41 <WarningLevel>4</WarningLevel>
41 <WarningLevel>4</WarningLevel>
42 <RunCodeAnalysis>true</RunCodeAnalysis>
42 <RunCodeAnalysis>true</RunCodeAnalysis>
43 <ConsolePause>false</ConsolePause>
43 <ConsolePause>false</ConsolePause>
44 </PropertyGroup>
44 </PropertyGroup>
45 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release 4.5|AnyCPU' ">
45 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release 4.5|AnyCPU' ">
46 <Optimize>true</Optimize>
46 <Optimize>true</Optimize>
47 <OutputPath>bin\Release</OutputPath>
47 <OutputPath>bin\Release</OutputPath>
48 <ErrorReport>prompt</ErrorReport>
48 <ErrorReport>prompt</ErrorReport>
49 <WarningLevel>4</WarningLevel>
49 <WarningLevel>4</WarningLevel>
50 <ConsolePause>false</ConsolePause>
50 <ConsolePause>false</ConsolePause>
51 <DefineConstants>NET_4_5</DefineConstants>
51 <DefineConstants>NET_4_5</DefineConstants>
52 </PropertyGroup>
52 </PropertyGroup>
53 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'DebugMono|AnyCPU' ">
53 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'DebugMono|AnyCPU' ">
54 <DebugSymbols>true</DebugSymbols>
54 <DebugSymbols>true</DebugSymbols>
55 <DebugType>full</DebugType>
55 <DebugType>full</DebugType>
56 <Optimize>false</Optimize>
56 <Optimize>false</Optimize>
57 <OutputPath>bin\Debug</OutputPath>
57 <OutputPath>bin\Debug</OutputPath>
58 <DefineConstants>TRACE;DEBUG;NET_4_5;MONO</DefineConstants>
58 <DefineConstants>TRACE;DEBUG;NET_4_5;MONO</DefineConstants>
59 <ErrorReport>prompt</ErrorReport>
59 <ErrorReport>prompt</ErrorReport>
60 <WarningLevel>4</WarningLevel>
60 <WarningLevel>4</WarningLevel>
61 <RunCodeAnalysis>true</RunCodeAnalysis>
61 <RunCodeAnalysis>true</RunCodeAnalysis>
62 <ConsolePause>false</ConsolePause>
62 <ConsolePause>false</ConsolePause>
63 </PropertyGroup>
63 </PropertyGroup>
64 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseMono|AnyCPU' ">
64 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseMono|AnyCPU' ">
65 <Optimize>true</Optimize>
65 <Optimize>true</Optimize>
66 <OutputPath>bin\Release</OutputPath>
66 <OutputPath>bin\Release</OutputPath>
67 <DefineConstants>NET_4_5;MONO;</DefineConstants>
67 <DefineConstants>NET_4_5;MONO;</DefineConstants>
68 <ErrorReport>prompt</ErrorReport>
68 <ErrorReport>prompt</ErrorReport>
69 <WarningLevel>4</WarningLevel>
69 <WarningLevel>4</WarningLevel>
70 <ConsolePause>false</ConsolePause>
70 <ConsolePause>false</ConsolePause>
71 </PropertyGroup>
71 </PropertyGroup>
72 <ItemGroup>
72 <ItemGroup>
73 <Reference Include="System" />
73 <Reference Include="System" />
74 <Reference Include="System.Xml" />
74 <Reference Include="System.Xml" />
75 <Reference Include="mscorlib" />
75 <Reference Include="mscorlib" />
76 </ItemGroup>
76 </ItemGroup>
77 <ItemGroup>
77 <ItemGroup>
78 <Compile Include="CustomEqualityComparer.cs" />
78 <Compile Include="CustomEqualityComparer.cs" />
79 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
79 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
80 <Compile Include="Diagnostics\EventText.cs" />
80 <Compile Include="Diagnostics\EventText.cs" />
81 <Compile Include="Diagnostics\LogChannel.cs" />
81 <Compile Include="Diagnostics\LogChannel.cs" />
82 <Compile Include="Diagnostics\LogicalOperation.cs" />
82 <Compile Include="Diagnostics\LogicalOperation.cs" />
83 <Compile Include="Diagnostics\TextFileListener.cs" />
83 <Compile Include="Diagnostics\TextFileListener.cs" />
84 <Compile Include="Diagnostics\TraceLog.cs" />
84 <Compile Include="Diagnostics\TraceLog.cs" />
85 <Compile Include="Diagnostics\TraceEvent.cs" />
85 <Compile Include="Diagnostics\TraceEvent.cs" />
86 <Compile Include="Diagnostics\TraceEventType.cs" />
86 <Compile Include="Diagnostics\TraceEventType.cs" />
87 <Compile Include="ICancellable.cs" />
87 <Compile Include="ICancellable.cs" />
88 <Compile Include="IProgressHandler.cs" />
88 <Compile Include="IProgressHandler.cs" />
89 <Compile Include="IProgressNotifier.cs" />
89 <Compile Include="IProgressNotifier.cs" />
90 <Compile Include="IPromiseT.cs" />
90 <Compile Include="IPromiseT.cs" />
91 <Compile Include="IPromise.cs" />
91 <Compile Include="IPromise.cs" />
92 <Compile Include="IServiceLocator.cs" />
92 <Compile Include="IServiceLocator.cs" />
93 <Compile Include="ITaskController.cs" />
93 <Compile Include="ITaskController.cs" />
94 <Compile Include="Parallels\DispatchPool.cs" />
94 <Compile Include="Parallels\DispatchPool.cs" />
95 <Compile Include="Parallels\ArrayTraits.cs" />
95 <Compile Include="Parallels\ArrayTraits.cs" />
96 <Compile Include="Parallels\MTQueue.cs" />
96 <Compile Include="Parallels\MTQueue.cs" />
97 <Compile Include="Parallels\WorkerPool.cs" />
97 <Compile Include="Parallels\WorkerPool.cs" />
98 <Compile Include="ProgressInitEventArgs.cs" />
98 <Compile Include="ProgressInitEventArgs.cs" />
99 <Compile Include="Properties\AssemblyInfo.cs" />
99 <Compile Include="Properties\AssemblyInfo.cs" />
100 <Compile Include="Parallels\AsyncPool.cs" />
100 <Compile Include="Parallels\AsyncPool.cs" />
101 <Compile Include="Safe.cs" />
101 <Compile Include="Safe.cs" />
102 <Compile Include="ValueEventArgs.cs" />
102 <Compile Include="ValueEventArgs.cs" />
103 <Compile Include="PromiseExtensions.cs" />
103 <Compile Include="PromiseExtensions.cs" />
104 <Compile Include="SyncContextPromise.cs" />
104 <Compile Include="SyncContextPromise.cs" />
105 <Compile Include="Diagnostics\OperationContext.cs" />
105 <Compile Include="Diagnostics\OperationContext.cs" />
106 <Compile Include="Diagnostics\TraceContext.cs" />
106 <Compile Include="Diagnostics\TraceContext.cs" />
107 <Compile Include="Diagnostics\LogEventArgs.cs" />
107 <Compile Include="Diagnostics\LogEventArgs.cs" />
108 <Compile Include="Diagnostics\LogEventArgsT.cs" />
108 <Compile Include="Diagnostics\LogEventArgsT.cs" />
109 <Compile Include="Diagnostics\Extensions.cs" />
109 <Compile Include="Diagnostics\Extensions.cs" />
110 <Compile Include="PromiseEventType.cs" />
110 <Compile Include="PromiseEventType.cs" />
111 <Compile Include="Parallels\AsyncQueue.cs" />
111 <Compile Include="Parallels\AsyncQueue.cs" />
112 <Compile Include="PromiseT.cs" />
112 <Compile Include="PromiseT.cs" />
113 <Compile Include="IDeferred.cs" />
113 <Compile Include="IDeferred.cs" />
114 <Compile Include="IDeferredT.cs" />
114 <Compile Include="IDeferredT.cs" />
115 <Compile Include="Promise.cs" />
115 <Compile Include="Promise.cs" />
116 <Compile Include="PromiseTransientException.cs" />
116 <Compile Include="PromiseTransientException.cs" />
117 <Compile Include="Parallels\Signal.cs" />
117 <Compile Include="Parallels\Signal.cs" />
118 <Compile Include="Parallels\SharedLock.cs" />
118 <Compile Include="Parallels\SharedLock.cs" />
119 <Compile Include="Diagnostics\ILogWriter.cs" />
119 <Compile Include="Diagnostics\ILogWriter.cs" />
120 <Compile Include="Diagnostics\ListenerBase.cs" />
120 <Compile Include="Diagnostics\ListenerBase.cs" />
121 <Compile Include="Parallels\BlockingQueue.cs" />
121 <Compile Include="Parallels\BlockingQueue.cs" />
122 <Compile Include="AbstractEvent.cs" />
122 <Compile Include="AbstractEvent.cs" />
123 <Compile Include="AbstractPromise.cs" />
123 <Compile Include="AbstractPromise.cs" />
124 <Compile Include="AbstractPromiseT.cs" />
124 <Compile Include="AbstractPromiseT.cs" />
125 <Compile Include="FuncTask.cs" />
125 <Compile Include="FuncTask.cs" />
126 <Compile Include="FuncTaskBase.cs" />
126 <Compile Include="FuncTaskBase.cs" />
127 <Compile Include="FuncTaskT.cs" />
127 <Compile Include="FuncTaskT.cs" />
128 <Compile Include="ActionChainTaskBase.cs" />
128 <Compile Include="ActionChainTaskBase.cs" />
129 <Compile Include="ActionChainTask.cs" />
129 <Compile Include="ActionChainTask.cs" />
130 <Compile Include="ActionChainTaskT.cs" />
130 <Compile Include="ActionChainTaskT.cs" />
131 <Compile Include="FuncChainTaskBase.cs" />
131 <Compile Include="FuncChainTaskBase.cs" />
132 <Compile Include="FuncChainTask.cs" />
132 <Compile Include="FuncChainTask.cs" />
133 <Compile Include="FuncChainTaskT.cs" />
133 <Compile Include="FuncChainTaskT.cs" />
134 <Compile Include="ActionTaskBase.cs" />
134 <Compile Include="ActionTaskBase.cs" />
135 <Compile Include="ActionTask.cs" />
135 <Compile Include="ActionTask.cs" />
136 <Compile Include="ActionTaskT.cs" />
136 <Compile Include="ActionTaskT.cs" />
137 <Compile Include="ICancellationToken.cs" />
137 <Compile Include="ICancellationToken.cs" />
138 <Compile Include="SuccessPromise.cs" />
138 <Compile Include="SuccessPromise.cs" />
139 <Compile Include="SuccessPromiseT.cs" />
139 <Compile Include="SuccessPromiseT.cs" />
140 <Compile Include="PromiseAwaiterT.cs" />
140 <Compile Include="PromiseAwaiterT.cs" />
141 <Compile Include="PromiseAwaiter.cs" />
141 <Compile Include="PromiseAwaiter.cs" />
142 <Compile Include="Components\ComponentContainer.cs" />
142 <Compile Include="Components\ComponentContainer.cs" />
143 <Compile Include="Components\Disposable.cs" />
143 <Compile Include="Components\Disposable.cs" />
144 <Compile Include="Components\DisposablePool.cs" />
144 <Compile Include="Components\DisposablePool.cs" />
145 <Compile Include="Components\ObjectPool.cs" />
145 <Compile Include="Components\ObjectPool.cs" />
146 <Compile Include="Components\ServiceLocator.cs" />
146 <Compile Include="Components\ServiceLocator.cs" />
147 <Compile Include="Components\IInitializable.cs" />
147 <Compile Include="Components\IInitializable.cs" />
148 <Compile Include="TaskController.cs" />
148 <Compile Include="TaskController.cs" />
149 <Compile Include="Components\App.cs" />
149 <Compile Include="Components\App.cs" />
150 <Compile Include="Components\IRunnable.cs" />
150 <Compile Include="Components\IRunnable.cs" />
151 <Compile Include="Components\ExecutionState.cs" />
151 <Compile Include="Components\ExecutionState.cs" />
152 <Compile Include="Components\RunnableComponent.cs" />
152 <Compile Include="Components\RunnableComponent.cs" />
153 <Compile Include="Components\IFactory.cs" />
153 <Compile Include="Components\IFactory.cs" />
154 <Compile Include="Automaton\DFAStateDescriptor.cs" />
155 <Compile Include="Automaton\EnumAlphabet.cs" />
154 <Compile Include="Automaton\EnumAlphabet.cs" />
156 <Compile Include="Automaton\IAlphabet.cs" />
155 <Compile Include="Automaton\IAlphabet.cs" />
157 <Compile Include="Automaton\ParserException.cs" />
156 <Compile Include="Automaton\ParserException.cs" />
158 <Compile Include="Automaton\Scanner.cs" />
159 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
157 <Compile Include="Automaton\IndexedAlphabetBase.cs" />
160 <Compile Include="Automaton\IAlphabetBuilder.cs" />
158 <Compile Include="Automaton\IAlphabetBuilder.cs" />
161 <Compile Include="Automaton\RegularExpressions\AltToken.cs" />
159 <Compile Include="Automaton\RegularExpressions\AltToken.cs" />
162 <Compile Include="Automaton\RegularExpressions\BinaryToken.cs" />
160 <Compile Include="Automaton\RegularExpressions\BinaryToken.cs" />
163 <Compile Include="Automaton\RegularExpressions\CatToken.cs" />
161 <Compile Include="Automaton\RegularExpressions\CatToken.cs" />
164 <Compile Include="Automaton\DFAConst.cs" />
162 <Compile Include="Automaton\DFAConst.cs" />
165 <Compile Include="Automaton\RegularExpressions\Grammar.cs" />
163 <Compile Include="Automaton\RegularExpressions\Grammar.cs" />
166 <Compile Include="Automaton\RegularExpressions\StarToken.cs" />
164 <Compile Include="Automaton\RegularExpressions\StarToken.cs" />
167 <Compile Include="Automaton\RegularExpressions\SymbolToken.cs" />
165 <Compile Include="Automaton\RegularExpressions\SymbolToken.cs" />
168 <Compile Include="Automaton\RegularExpressions\EmptyToken.cs" />
166 <Compile Include="Automaton\RegularExpressions\EmptyToken.cs" />
169 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
167 <Compile Include="Automaton\RegularExpressions\EndToken.cs" />
170 <Compile Include="Automaton\RegularExpressions\Token.cs" />
168 <Compile Include="Automaton\RegularExpressions\Token.cs" />
171 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
169 <Compile Include="Automaton\RegularExpressions\IVisitor.cs" />
172 <Compile Include="Automaton\AutomatonTransition.cs" />
170 <Compile Include="Automaton\AutomatonTransition.cs" />
173 <Compile Include="Formats\JSON\JSONElementContext.cs" />
171 <Compile Include="Formats\JSON\JSONElementContext.cs" />
174 <Compile Include="Formats\JSON\JSONElementType.cs" />
172 <Compile Include="Formats\JSON\JSONElementType.cs" />
175 <Compile Include="Formats\JSON\JSONGrammar.cs" />
173 <Compile Include="Formats\JSON\JSONGrammar.cs" />
176 <Compile Include="Formats\JSON\JSONParser.cs" />
174 <Compile Include="Formats\JSON\JSONParser.cs" />
177 <Compile Include="Formats\JSON\JSONScanner.cs" />
175 <Compile Include="Formats\JSON\JSONScanner.cs" />
178 <Compile Include="Formats\JSON\JsonTokenType.cs" />
176 <Compile Include="Formats\JSON\JsonTokenType.cs" />
179 <Compile Include="Formats\JSON\JSONWriter.cs" />
177 <Compile Include="Formats\JSON\JSONWriter.cs" />
180 <Compile Include="Formats\JSON\JSONXmlReader.cs" />
178 <Compile Include="Formats\JSON\JSONXmlReader.cs" />
181 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
179 <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
182 <Compile Include="Formats\JSON\StringTranslator.cs" />
180 <Compile Include="Formats\JSON\StringTranslator.cs" />
183 <Compile Include="Automaton\MapAlphabet.cs" />
181 <Compile Include="Automaton\MapAlphabet.cs" />
184 <Compile Include="Automaton\DummyAlphabet.cs" />
182 <Compile Include="Automaton\DummyAlphabet.cs" />
185 <Compile Include="Formats\CharAlphabet.cs" />
183 <Compile Include="Formats\CharAlphabet.cs" />
186 <Compile Include="Formats\ByteAlphabet.cs" />
184 <Compile Include="Formats\ByteAlphabet.cs" />
187 <Compile Include="Automaton\IDFATable.cs" />
185 <Compile Include="Automaton\IDFATable.cs" />
188 <Compile Include="Automaton\IDFATableBuilder.cs" />
186 <Compile Include="Automaton\IDFATableBuilder.cs" />
189 <Compile Include="Automaton\DFATable.cs" />
187 <Compile Include="Automaton\DFATable.cs" />
190 <Compile Include="Automaton\RegularExpressions\RegularDFA.cs" />
188 <Compile Include="Automaton\RegularExpressions\RegularDFA.cs" />
191 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
189 <Compile Include="Automaton\RegularExpressions\RegularExpressionVisitor.cs" />
192 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
190 <Compile Include="Automaton\RegularExpressions\ITaggedDFABuilder.cs" />
193 <Compile Include="Automaton\RegularExpressions\DFAStateDescriptorT.cs" />
194 <Compile Include="Formats\BufferScanner.cs" />
195 <Compile Include="Formats\TextScanner.cs" />
191 <Compile Include="Formats\TextScanner.cs" />
192 <Compile Include="Formats\StringScanner.cs" />
193 <Compile Include="Formats\ReaderScanner.cs" />
194 <Compile Include="Formats\ScannerContext.cs" />
196 </ItemGroup>
195 </ItemGroup>
197 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
196 <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
198 <ItemGroup />
197 <ItemGroup />
199 <ProjectExtensions>
198 <ProjectExtensions>
200 <MonoDevelop>
199 <MonoDevelop>
201 <Properties>
200 <Properties>
202 <Policies>
201 <Policies>
203 <CSharpFormattingPolicy IndentSwitchBody="True" NamespaceBraceStyle="EndOfLine" ClassBraceStyle="EndOfLine" InterfaceBraceStyle="EndOfLine" StructBraceStyle="EndOfLine" EnumBraceStyle="EndOfLine" MethodBraceStyle="EndOfLine" ConstructorBraceStyle="EndOfLine" DestructorBraceStyle="EndOfLine" BeforeMethodDeclarationParentheses="False" BeforeMethodCallParentheses="False" BeforeConstructorDeclarationParentheses="False" NewLineBeforeConstructorInitializerColon="NewLine" NewLineAfterConstructorInitializerColon="SameLine" BeforeIndexerDeclarationBracket="False" BeforeDelegateDeclarationParentheses="False" NewParentheses="False" SpacesBeforeBrackets="False" inheritsSet="Mono" inheritsScope="text/x-csharp" scope="text/x-csharp" />
202 <CSharpFormattingPolicy IndentSwitchBody="True" NamespaceBraceStyle="EndOfLine" ClassBraceStyle="EndOfLine" InterfaceBraceStyle="EndOfLine" StructBraceStyle="EndOfLine" EnumBraceStyle="EndOfLine" MethodBraceStyle="EndOfLine" ConstructorBraceStyle="EndOfLine" DestructorBraceStyle="EndOfLine" BeforeMethodDeclarationParentheses="False" BeforeMethodCallParentheses="False" BeforeConstructorDeclarationParentheses="False" NewLineBeforeConstructorInitializerColon="NewLine" NewLineAfterConstructorInitializerColon="SameLine" BeforeIndexerDeclarationBracket="False" BeforeDelegateDeclarationParentheses="False" NewParentheses="False" SpacesBeforeBrackets="False" inheritsSet="Mono" inheritsScope="text/x-csharp" scope="text/x-csharp" />
204 <TextStylePolicy FileWidth="120" EolMarker="Unix" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/x-csharp" />
203 <TextStylePolicy FileWidth="120" EolMarker="Unix" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/x-csharp" />
205 <DotNetNamingPolicy DirectoryNamespaceAssociation="PrefixedHierarchical" ResourceNamePolicy="MSBuild" />
204 <DotNetNamingPolicy DirectoryNamespaceAssociation="PrefixedHierarchical" ResourceNamePolicy="MSBuild" />
206 <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="application/xml" />
205 <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="application/xml" />
207 <XmlFormattingPolicy inheritsSet="Mono" inheritsScope="application/xml" scope="application/xml" />
206 <XmlFormattingPolicy inheritsSet="Mono" inheritsScope="application/xml" scope="application/xml" />
208 <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/plain" />
207 <TextStylePolicy FileWidth="120" TabsToSpaces="False" inheritsSet="VisualStudio" inheritsScope="text/plain" scope="text/plain" />
209 <NameConventionPolicy>
208 <NameConventionPolicy>
210 <Rules>
209 <Rules>
211 <NamingRule Name="Namespaces" AffectedEntity="Namespace" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
210 <NamingRule Name="Namespaces" AffectedEntity="Namespace" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
212 <NamingRule Name="Types" AffectedEntity="Class, Struct, Enum, Delegate" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
211 <NamingRule Name="Types" AffectedEntity="Class, Struct, Enum, Delegate" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
213 <NamingRule Name="Interfaces" AffectedEntity="Interface" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
212 <NamingRule Name="Interfaces" AffectedEntity="Interface" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
214 <RequiredPrefixes>
213 <RequiredPrefixes>
215 <String>I</String>
214 <String>I</String>
216 </RequiredPrefixes>
215 </RequiredPrefixes>
217 </NamingRule>
216 </NamingRule>
218 <NamingRule Name="Attributes" AffectedEntity="CustomAttributes" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
217 <NamingRule Name="Attributes" AffectedEntity="CustomAttributes" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
219 <RequiredSuffixes>
218 <RequiredSuffixes>
220 <String>Attribute</String>
219 <String>Attribute</String>
221 </RequiredSuffixes>
220 </RequiredSuffixes>
222 </NamingRule>
221 </NamingRule>
223 <NamingRule Name="Event Arguments" AffectedEntity="CustomEventArgs" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
222 <NamingRule Name="Event Arguments" AffectedEntity="CustomEventArgs" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
224 <RequiredSuffixes>
223 <RequiredSuffixes>
225 <String>EventArgs</String>
224 <String>EventArgs</String>
226 </RequiredSuffixes>
225 </RequiredSuffixes>
227 </NamingRule>
226 </NamingRule>
228 <NamingRule Name="Exceptions" AffectedEntity="CustomExceptions" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
227 <NamingRule Name="Exceptions" AffectedEntity="CustomExceptions" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
229 <RequiredSuffixes>
228 <RequiredSuffixes>
230 <String>Exception</String>
229 <String>Exception</String>
231 </RequiredSuffixes>
230 </RequiredSuffixes>
232 </NamingRule>
231 </NamingRule>
233 <NamingRule Name="Methods" AffectedEntity="Methods" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
232 <NamingRule Name="Methods" AffectedEntity="Methods" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
234 <NamingRule Name="Static Readonly Fields" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Protected, Public" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True" />
233 <NamingRule Name="Static Readonly Fields" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Protected, Public" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True" />
235 <NamingRule Name="Fields (Non Private)" AffectedEntity="Field" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
234 <NamingRule Name="Fields (Non Private)" AffectedEntity="Field" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
236 <NamingRule Name="ReadOnly Fields (Non Private)" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False" />
235 <NamingRule Name="ReadOnly Fields (Non Private)" AffectedEntity="ReadonlyField" VisibilityMask="Internal, Public" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False" />
237 <NamingRule Name="Fields (Private)" AffectedEntity="Field, ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
236 <NamingRule Name="Fields (Private)" AffectedEntity="Field, ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
238 <RequiredPrefixes>
237 <RequiredPrefixes>
239 <String>m_</String>
238 <String>m_</String>
240 </RequiredPrefixes>
239 </RequiredPrefixes>
241 </NamingRule>
240 </NamingRule>
242 <NamingRule Name="Static Fields (Private)" AffectedEntity="Field" VisibilityMask="Private" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True">
241 <NamingRule Name="Static Fields (Private)" AffectedEntity="Field" VisibilityMask="Private" NamingStyle="CamelCase" IncludeInstanceMembers="False" IncludeStaticEntities="True">
243 <RequiredPrefixes>
242 <RequiredPrefixes>
244 <String>_</String>
243 <String>_</String>
245 </RequiredPrefixes>
244 </RequiredPrefixes>
246 </NamingRule>
245 </NamingRule>
247 <NamingRule Name="ReadOnly Fields (Private)" AffectedEntity="ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
246 <NamingRule Name="ReadOnly Fields (Private)" AffectedEntity="ReadonlyField" VisibilityMask="Private, Protected" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="False">
248 <RequiredPrefixes>
247 <RequiredPrefixes>
249 <String>m_</String>
248 <String>m_</String>
250 </RequiredPrefixes>
249 </RequiredPrefixes>
251 </NamingRule>
250 </NamingRule>
252 <NamingRule Name="Constant Fields" AffectedEntity="ConstantField" VisibilityMask="VisibilityMask" NamingStyle="AllUpper" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
251 <NamingRule Name="Constant Fields" AffectedEntity="ConstantField" VisibilityMask="VisibilityMask" NamingStyle="AllUpper" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
253 <NamingRule Name="Properties" AffectedEntity="Property" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
252 <NamingRule Name="Properties" AffectedEntity="Property" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
254 <NamingRule Name="Events" AffectedEntity="Event" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
253 <NamingRule Name="Events" AffectedEntity="Event" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
255 <NamingRule Name="Enum Members" AffectedEntity="EnumMember" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
254 <NamingRule Name="Enum Members" AffectedEntity="EnumMember" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
256 <NamingRule Name="Parameters" AffectedEntity="Parameter, LocalVariable" VisibilityMask="VisibilityMask" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
255 <NamingRule Name="Parameters" AffectedEntity="Parameter, LocalVariable" VisibilityMask="VisibilityMask" NamingStyle="CamelCase" IncludeInstanceMembers="True" IncludeStaticEntities="True" />
257 <NamingRule Name="Type Parameters" AffectedEntity="TypeParameter" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
256 <NamingRule Name="Type Parameters" AffectedEntity="TypeParameter" VisibilityMask="VisibilityMask" NamingStyle="PascalCase" IncludeInstanceMembers="True" IncludeStaticEntities="True">
258 <RequiredPrefixes>
257 <RequiredPrefixes>
259 <String>T</String>
258 <String>T</String>
260 </RequiredPrefixes>
259 </RequiredPrefixes>
261 </NamingRule>
260 </NamingRule>
262 </Rules>
261 </Rules>
263 </NameConventionPolicy>
262 </NameConventionPolicy>
264 </Policies>
263 </Policies>
265 </Properties>
264 </Properties>
266 </MonoDevelop>
265 </MonoDevelop>
267 </ProjectExtensions>
266 </ProjectExtensions>
268 <ItemGroup>
267 <ItemGroup>
269 <Folder Include="Components\" />
268 <Folder Include="Components\" />
270 <Folder Include="Automaton\RegularExpressions\" />
269 <Folder Include="Automaton\RegularExpressions\" />
271 <Folder Include="Formats\" />
270 <Folder Include="Formats\" />
272 <Folder Include="Formats\JSON\" />
271 <Folder Include="Formats\JSON\" />
273 </ItemGroup>
272 </ItemGroup>
274 </Project> No newline at end of file
273 </Project>
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now