This diff has been collapsed as it changes many lines, (510 lines changed) Show them Hide them | |||||
@@ -1,255 +1,255 | |||||
1 | using Implab; |
|
1 | using Implab; | |
2 | using System; |
|
2 | using System; | |
3 | using System.Collections.Generic; |
|
3 | using System.Collections.Generic; | |
4 | using System.IO; |
|
4 | using System.IO; | |
5 | using Implab.Components; |
|
5 | using Implab.Components; | |
6 | using Implab.Automaton.RegularExpressions; |
|
6 | using Implab.Automaton.RegularExpressions; | |
7 |
|
7 | |||
8 | namespace Implab.Automaton { |
|
8 | namespace Implab.Automaton { | |
9 | /// <summary> |
|
9 | /// <summary> | |
10 | /// ΠΠ°Π·ΠΎΠ²ΡΠΉ ΠΊΠ»Π°ΡΡ Π΄Π»Ρ ΡΠ°Π·Π±ΠΎΡΠ° ΠΏΠΎΡΠΎΠΊΠ° Π²Ρ ΠΎΠ΄Π½ΡΡ ΡΠΈΠΌΠ²ΠΎΠ»ΠΎΠ² Π½Π° ΡΠΎΠΊΠ΅Π½Ρ. |
|
10 | /// ΠΠ°Π·ΠΎΠ²ΡΠΉ ΠΊΠ»Π°ΡΡ Π΄Π»Ρ ΡΠ°Π·Π±ΠΎΡΠ° ΠΏΠΎΡΠΎΠΊΠ° Π²Ρ ΠΎΠ΄Π½ΡΡ ΡΠΈΠΌΠ²ΠΎΠ»ΠΎΠ² Π½Π° ΡΠΎΠΊΠ΅Π½Ρ. | |
11 | /// </summary> |
|
11 | /// </summary> | |
12 | /// <remarks> |
|
12 | /// <remarks> | |
13 | /// Π‘ΠΊΠ°Π½Π½Π΅Ρ ΠΈΠΌΠ΅Π΅Ρ Π²Π½ΡΡΡΠΈ Π±ΡΡΡΠ΅Ρ Ρ ΡΠΈΠΌΠΎΠ»Π°ΠΌΠΈ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ ΡΠ΅ΠΊΡΡΠ°, ΠΏΠΎ ΠΊΠΎΡΠΎΡΠΎΠΌΡ ΠΏΠ΅ΡΠ΅ΠΌΠ΅ΡΠ°ΡΡΡΡ Π΄Π²Π° |
|
13 | /// Π‘ΠΊΠ°Π½Π½Π΅Ρ ΠΈΠΌΠ΅Π΅Ρ Π²Π½ΡΡΡΠΈ Π±ΡΡΡΠ΅Ρ Ρ ΡΠΈΠΌΠΎΠ»Π°ΠΌΠΈ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ ΡΠ΅ΠΊΡΡΠ°, ΠΏΠΎ ΠΊΠΎΡΠΎΡΠΎΠΌΡ ΠΏΠ΅ΡΠ΅ΠΌΠ΅ΡΠ°ΡΡΡΡ Π΄Π²Π° | |
14 | /// ΡΠΊΠ°Π·Π°ΡΠ΅Π»Ρ, Π½Π°ΡΠ°Π»Π° ΠΈ ΠΊΠΎΠ½ΡΠ° ΡΠΎΠΊΠ΅Π½Π°, ΠΏΡΠΈ ΠΏΠ΅ΡΠ΅ΠΌΠ΅ΡΠ΅Π½ΠΈΠΈ ΠΈΡΠΊΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ ΠΠΠ Π΄Π»Ρ ΠΎΠΏΡΠ΅Π΄Π΅Π»Π΅Π½ΠΈΡ |
|
14 | /// ΡΠΊΠ°Π·Π°ΡΠ΅Π»Ρ, Π½Π°ΡΠ°Π»Π° ΠΈ ΠΊΠΎΠ½ΡΠ° ΡΠΎΠΊΠ΅Π½Π°, ΠΏΡΠΈ ΠΏΠ΅ΡΠ΅ΠΌΠ΅ΡΠ΅Π½ΠΈΠΈ ΠΈΡΠΊΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ ΠΠΠ Π΄Π»Ρ ΠΎΠΏΡΠ΅Π΄Π΅Π»Π΅Π½ΠΈΡ | |
15 | /// ΠΊΠΎΠ½ΡΠ° ΡΠΎΠΊΠ΅Π½Π° ΠΈ Π΄ΠΎΠΏΡΡΡΠΈΠΌΠΎΡΡΠΈ ΡΠ΅ΠΊΡΡΠ΅Π³ΠΎ ΡΠΈΠΌΠ²ΠΎΠ»Π°. |
|
15 | /// ΠΊΠΎΠ½ΡΠ° ΡΠΎΠΊΠ΅Π½Π° ΠΈ Π΄ΠΎΠΏΡΡΡΠΈΠΌΠΎΡΡΠΈ ΡΠ΅ΠΊΡΡΠ΅Π³ΠΎ ΡΠΈΠΌΠ²ΠΎΠ»Π°. | |
16 | /// </remarks> |
|
16 | /// </remarks> | |
17 | public abstract class Scanner<TTag> : Disposable { |
|
17 | public abstract class Scanner<TTag> : Disposable { | |
18 | protected struct ScannerConfig { |
|
18 | protected struct ScannerConfig { | |
19 | public readonly DFAStateDescriptor<TTag>[] states; |
|
19 | public readonly DFAStateDescriptor<TTag>[] states; | |
20 | public readonly int[] alphabet; |
|
20 | public readonly int[] alphabet; | |
21 | public readonly int initialState; |
|
21 | public readonly int initialState; | |
22 |
|
22 | |||
23 | public ScannerConfig(DFAStateDescriptor<TTag>[] states, int[] alphabet, int initialState) { |
|
23 | public ScannerConfig(DFAStateDescriptor<TTag>[] states, int[] alphabet, int initialState) { | |
24 | this.initialState = initialState; |
|
24 | this.initialState = initialState; | |
25 | this.alphabet = alphabet; |
|
25 | this.alphabet = alphabet; | |
26 | this.states = states; |
|
26 | this.states = states; | |
27 | } |
|
27 | } | |
28 | } |
|
28 | } | |
29 |
|
29 | |||
30 | Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>(); |
|
30 | Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>(); | |
31 |
|
31 | |||
32 | ScannerConfig m_config; |
|
32 | ScannerConfig m_config; | |
33 |
|
33 | |||
34 | protected DFAStateDescriptor<TTag> m_currentState; |
|
34 | protected DFAStateDescriptor<TTag> m_currentState; | |
35 | int m_previewCode; |
|
35 | int m_previewCode; | |
36 |
|
36 | |||
37 | protected int m_tokenLen; |
|
37 | protected int m_tokenLen; | |
38 | protected int m_tokenOffset; |
|
38 | protected int m_tokenOffset; | |
39 |
|
39 | |||
40 | protected char[] m_buffer; |
|
40 | protected char[] m_buffer; | |
41 | protected int m_bufferSize; |
|
41 | protected int m_bufferSize; | |
42 | protected int m_pointer; |
|
42 | protected int m_pointer; | |
43 |
|
43 | |||
44 | TextReader m_reader; |
|
44 | TextReader m_reader; | |
45 | bool m_disposeReader; |
|
45 | bool m_disposeReader; | |
46 | int m_chunkSize = 1024; // 1k |
|
46 | int m_chunkSize = 1024; // 1k | |
47 | int m_limit = 10 * 1024 * 1024; // 10Mb |
|
47 | int m_limit = 10 * 1024 * 1024; // 10Mb | |
48 |
|
48 | |||
49 | protected Scanner(ScannerConfig config) { |
|
49 | protected Scanner(ScannerConfig config) { | |
50 | Safe.ArgumentNotEmpty(config.states, "config.states"); |
|
50 | Safe.ArgumentNotEmpty(config.states, "config.states"); | |
51 | Safe.ArgumentNotNull(config.alphabet, "config.alphabet"); |
|
51 | Safe.ArgumentNotNull(config.alphabet, "config.alphabet"); | |
52 |
|
52 | |||
53 | m_config = config; |
|
53 | m_config = config; | |
54 | } |
|
54 | } | |
55 |
|
55 | |||
56 | /// <summary> |
|
56 | /// <summary> | |
57 | /// ΠΠ°ΠΏΠΎΠ»Π½ΡΠ΅Ρ Π²Ρ ΠΎΠ΄Π½ΡΠΌΠΈ Π΄Π°Π½Π½ΡΠΌΠΈ Π±ΡΡΡΠ΅Ρ. |
|
57 | /// ΠΠ°ΠΏΠΎΠ»Π½ΡΠ΅Ρ Π²Ρ ΠΎΠ΄Π½ΡΠΌΠΈ Π΄Π°Π½Π½ΡΠΌΠΈ Π±ΡΡΡΠ΅Ρ. | |
58 | /// </summary> |
|
58 | /// </summary> | |
59 | /// <param name="data">ΠΠ°Π½Π½ΡΠ΅ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> |
|
59 | /// <param name="data">ΠΠ°Π½Π½ΡΠ΅ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> | |
60 | /// <remarks>ΠΠΎΠΏΠΈΡΠΎΠ²Π°Π½ΠΈΠ΅ Π΄Π°Π½Π½ΡΡ Π½Π΅ ΠΏΡΠΎΠΈΡΡ ΠΎΠ΄ΠΈΡ, ΠΏΠ΅ΡΠ΅Π΄Π°Π½Π½ΡΠΉ ΠΌΠ°ΡΡΠΈΠ² ΠΈΡΠΏΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ Π² |
|
60 | /// <remarks>ΠΠΎΠΏΠΈΡΠΎΠ²Π°Π½ΠΈΠ΅ Π΄Π°Π½Π½ΡΡ Π½Π΅ ΠΏΡΠΎΠΈΡΡ ΠΎΠ΄ΠΈΡ, ΠΏΠ΅ΡΠ΅Π΄Π°Π½Π½ΡΠΉ ΠΌΠ°ΡΡΠΈΠ² ΠΈΡΠΏΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ Π² | |
61 | /// ΠΊΠ°ΡΠ΅ΡΡΠ²Π΅ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ Π±ΡΡΡΠ΅ΡΠ°.</remarks> |
|
61 | /// ΠΊΠ°ΡΠ΅ΡΡΠ²Π΅ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ Π±ΡΡΡΠ΅ΡΠ°.</remarks> | |
62 | public void Feed(char[] data) { |
|
62 | public void Feed(char[] data) { | |
63 | Safe.ArgumentNotNull(data, "data"); |
|
63 | Safe.ArgumentNotNull(data, "data"); | |
64 |
|
64 | |||
65 | Feed(data, data.Length); |
|
65 | Feed(data, data.Length); | |
66 | } |
|
66 | } | |
67 |
|
67 | |||
68 | /// <summary> |
|
68 | /// <summary> | |
69 | /// ΠΠ°ΠΏΠΎΠ»Π½ΡΠ΅Ρ Π±ΡΡΡΡΡ ΡΡΠ΅Π½ΠΈΡ Π²Ρ ΠΎΠ΄Π½ΡΠΌΠΈ Π΄Π°Π½Π½ΡΠΌΠΈ. |
|
69 | /// ΠΠ°ΠΏΠΎΠ»Π½ΡΠ΅Ρ Π±ΡΡΡΡΡ ΡΡΠ΅Π½ΠΈΡ Π²Ρ ΠΎΠ΄Π½ΡΠΌΠΈ Π΄Π°Π½Π½ΡΠΌΠΈ. | |
70 | /// </summary> |
|
70 | /// </summary> | |
71 | /// <param name="data">ΠΠ°Π½Π½ΡΠ΅ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> |
|
71 | /// <param name="data">ΠΠ°Π½Π½ΡΠ΅ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> | |
72 | /// <param name="length">ΠΠ»ΠΈΠ½Π° Π΄Π°Π½Π½ΡΡ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> |
|
72 | /// <param name="length">ΠΠ»ΠΈΠ½Π° Π΄Π°Π½Π½ΡΡ Π΄Π»Ρ ΠΎΠ±ΡΠ°Π±ΠΎΡΠΊΠΈ.</param> | |
73 | /// <remarks>ΠΠΎΠΏΠΈΡΠΎΠ²Π°Π½ΠΈΠ΅ Π΄Π°Π½Π½ΡΡ Π½Π΅ ΠΏΡΠΎΠΈΡΡ ΠΎΠ΄ΠΈΡ, ΠΏΠ΅ΡΠ΅Π΄Π°Π½Π½ΡΠΉ ΠΌΠ°ΡΡΠΈΠ² ΠΈΡΠΏΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ Π² |
|
73 | /// <remarks>ΠΠΎΠΏΠΈΡΠΎΠ²Π°Π½ΠΈΠ΅ Π΄Π°Π½Π½ΡΡ Π½Π΅ ΠΏΡΠΎΠΈΡΡ ΠΎΠ΄ΠΈΡ, ΠΏΠ΅ΡΠ΅Π΄Π°Π½Π½ΡΠΉ ΠΌΠ°ΡΡΠΈΠ² ΠΈΡΠΏΠΎΠ»ΡΠ·ΡΠ΅ΡΡΡ Π² | |
74 | /// ΠΊΠ°ΡΠ΅ΡΡΠ²Π΅ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ Π±ΡΡΡΠ΅ΡΠ°.</remarks> |
|
74 | /// ΠΊΠ°ΡΠ΅ΡΡΠ²Π΅ Π²Ρ ΠΎΠ΄Π½ΠΎΠ³ΠΎ Π±ΡΡΡΠ΅ΡΠ°.</remarks> | |
75 | public void Feed(char[] data, int length) { |
|
75 | public void Feed(char[] data, int length) { | |
76 | Safe.ArgumentNotNull(data, "data"); |
|
76 | Safe.ArgumentNotNull(data, "data"); | |
77 | Safe.ArgumentInRange(length, 0, data.Length, "length"); |
|
77 | Safe.ArgumentInRange(length, 0, data.Length, "length"); | |
78 | AssertNotDisposed(); |
|
78 | AssertNotDisposed(); | |
79 |
|
79 | |||
80 | m_pointer = -1; |
|
80 | m_pointer = -1; | |
81 | m_buffer = data; |
|
81 | m_buffer = data; | |
82 | m_bufferSize = length; |
|
82 | m_bufferSize = length; | |
83 | Shift(); |
|
83 | Shift(); | |
84 | } |
|
84 | } | |
85 |
|
85 | |||
86 | public void Feed(TextReader reader, bool dispose) { |
|
86 | public void Feed(TextReader reader, bool dispose) { | |
87 | Safe.ArgumentNotNull(reader, "reader"); |
|
87 | Safe.ArgumentNotNull(reader, "reader"); | |
88 | AssertNotDisposed(); |
|
88 | AssertNotDisposed(); | |
89 |
|
89 | |||
90 | if (m_reader != null && m_disposeReader) |
|
90 | if (m_reader != null && m_disposeReader) | |
91 | m_reader.Dispose(); |
|
91 | m_reader.Dispose(); | |
92 |
|
92 | |||
93 | m_reader = reader; |
|
93 | m_reader = reader; | |
94 | m_disposeReader = dispose; |
|
94 | m_disposeReader = dispose; | |
95 | m_pointer = -1; |
|
95 | m_pointer = -1; | |
96 | m_buffer = new char[m_chunkSize]; |
|
96 | m_buffer = new char[m_chunkSize]; | |
97 | m_bufferSize = 0; |
|
97 | m_bufferSize = 0; | |
98 | Shift(); |
|
98 | Shift(); | |
99 | } |
|
99 | } | |
100 |
|
100 | |||
101 | /// <summary> |
|
101 | /// <summary> | |
102 | /// ΠΠΎΠ»ΡΡΠ°Π΅Ρ ΡΠ΅ΠΊΡΡΠΈΠΉ ΡΠΎΠΊΠ΅Π½ Π² Π²ΠΈΠ΄Π΅ ΡΡΡΠΎΠΊΠΈ. |
|
102 | /// ΠΠΎΠ»ΡΡΠ°Π΅Ρ ΡΠ΅ΠΊΡΡΠΈΠΉ ΡΠΎΠΊΠ΅Π½ Π² Π²ΠΈΠ΄Π΅ ΡΡΡΠΎΠΊΠΈ. | |
103 | /// </summary> |
|
103 | /// </summary> | |
104 | /// <returns></returns> |
|
104 | /// <returns></returns> | |
105 | protected string GetTokenValue() { |
|
105 | protected string GetTokenValue() { | |
106 | return new String(m_buffer, m_tokenOffset, m_tokenLen); |
|
106 | return new String(m_buffer, m_tokenOffset, m_tokenLen); | |
107 | } |
|
107 | } | |
108 |
|
108 | |||
109 | /// <summary> |
|
109 | /// <summary> | |
110 | /// ΠΠ΅ΡΠΊΠΈ ΡΠ΅ΠΊΡΡΠ΅Π³ΠΎ ΡΠΎΠΊΠ΅Π½Π°, ΠΊΠΎΡΠΎΡΡΠ΅ Π±ΡΠ»ΠΈ Π½Π°Π·Π½Π°ΡΠ΅Π½Ρ Π² ΡΠ΅Π³ΡΠ»ΡΡΠ½ΠΎΠΌ Π²ΡΡΠ°ΠΆΠ΅Π½ΠΈΠΈ. |
|
110 | /// ΠΠ΅ΡΠΊΠΈ ΡΠ΅ΠΊΡΡΠ΅Π³ΠΎ ΡΠΎΠΊΠ΅Π½Π°, ΠΊΠΎΡΠΎΡΡΠ΅ Π±ΡΠ»ΠΈ Π½Π°Π·Π½Π°ΡΠ΅Π½Ρ Π² ΡΠ΅Π³ΡΠ»ΡΡΠ½ΠΎΠΌ Π²ΡΡΠ°ΠΆΠ΅Π½ΠΈΠΈ. | |
111 | /// </summary> |
|
111 | /// </summary> | |
112 | protected TTag[] TokenTags { |
|
112 | protected TTag[] TokenTags { | |
113 | get { |
|
113 | get { | |
114 | return m_currentState.tags; |
|
114 | return m_currentState.tags; | |
115 | } |
|
115 | } | |
116 | } |
|
116 | } | |
117 |
|
117 | |||
118 | /// <summary> |
|
118 | /// <summary> | |
119 | /// ΠΡΠΈΠ·Π½Π°ΠΊ ΠΊΠΎΠ½ΡΠ° Π΄Π°Π½Π½ΡΡ |
|
119 | /// ΠΡΠΈΠ·Π½Π°ΠΊ ΠΊΠΎΠ½ΡΠ° Π΄Π°Π½Π½ΡΡ | |
120 | /// </summary> |
|
120 | /// </summary> | |
121 | public bool EOF { |
|
121 | public bool EOF { | |
122 | get { |
|
122 | get { | |
123 | return m_pointer >= m_bufferSize; |
|
123 | return m_pointer >= m_bufferSize; | |
124 | } |
|
124 | } | |
125 | } |
|
125 | } | |
126 |
|
126 | |||
127 | /// <summary> |
|
127 | /// <summary> | |
128 | /// Π§ΠΈΡΠ°Π΅Ρ ΡΠ»Π΅Π΄ΡΡΡΠΈΠΉ ΡΠΎΠΊΠ΅Π½, ΠΏΡΠΈ ΡΡΠΎΠΌ <see cref="m_tokenOffset"/> ΡΠΊΠ°Π·ΡΠ²Π°Π΅Ρ Π½Π° Π½Π°ΡΠ°Π»ΠΎ ΡΠΎΠΊΠ΅Π½Π°, |
|
128 | /// Π§ΠΈΡΠ°Π΅Ρ ΡΠ»Π΅Π΄ΡΡΡΠΈΠΉ ΡΠΎΠΊΠ΅Π½, ΠΏΡΠΈ ΡΡΠΎΠΌ <see cref="m_tokenOffset"/> ΡΠΊΠ°Π·ΡΠ²Π°Π΅Ρ Π½Π° Π½Π°ΡΠ°Π»ΠΎ ΡΠΎΠΊΠ΅Π½Π°, | |
129 | /// <see cref="m_tokenLen"/> Π½Π° Π΄Π»ΠΈΠ½Ρ ΡΠΎΠΊΠ΅Π½Π°, <see cref="m_buffer"/> - ΠΌΠ°ΡΡΠΈΠ² ΡΠΈΠΌΠ²ΠΎΠ»ΠΎΠ², Π² |
|
129 | /// <see cref="m_tokenLen"/> Π½Π° Π΄Π»ΠΈΠ½Ρ ΡΠΎΠΊΠ΅Π½Π°, <see cref="m_buffer"/> - ΠΌΠ°ΡΡΠΈΠ² ΡΠΈΠΌΠ²ΠΎΠ»ΠΎΠ², Π² | |
130 | /// ΠΊΠΎΡΠΎΡΠΎΠΌ Π½Π°Ρ ΠΎΠ΄ΠΈΡΡΡ ΡΠΎΠΊΠ΅Π½. |
|
130 | /// ΠΊΠΎΡΠΎΡΠΎΠΌ Π½Π°Ρ ΠΎΠ΄ΠΈΡΡΡ ΡΠΎΠΊΠ΅Π½. | |
131 | /// </summary> |
|
131 | /// </summary> | |
132 | /// <returns><c>false</c> - Π΄ΠΎΡΡΠΈΠ³Π½ΡΡ ΠΊΠΎΠ½Π΅Ρ Π΄Π°Π½Π½ΡΡ , ΡΠΎΠΊΠ΅Π½ Π½Π΅ ΠΏΡΠΎΡΠΈΡΠ°Π½.</returns> |
|
132 | /// <returns><c>false</c> - Π΄ΠΎΡΡΠΈΠ³Π½ΡΡ ΠΊΠΎΠ½Π΅Ρ Π΄Π°Π½Π½ΡΡ , ΡΠΎΠΊΠ΅Π½ Π½Π΅ ΠΏΡΠΎΡΠΈΡΠ°Π½.</returns> | |
133 | protected bool ReadTokenInternal() { |
|
133 | protected bool ReadTokenInternal() { | |
134 | if (m_pointer >= m_bufferSize) |
|
134 | if (m_pointer >= m_bufferSize) | |
135 | return false; |
|
135 | return false; | |
136 |
|
136 | |||
137 | m_currentState = m_config.states[m_config.initialState]; |
|
137 | m_currentState = m_config.states[m_config.initialState]; | |
138 | m_tokenLen = 0; |
|
138 | m_tokenLen = 0; | |
139 | m_tokenOffset = m_pointer; |
|
139 | m_tokenOffset = m_pointer; | |
140 | int nextState; |
|
140 | int nextState; | |
141 | do { |
|
141 | do { | |
142 | nextState = m_currentState.transitions[m_previewCode]; |
|
142 | nextState = m_currentState.transitions[m_previewCode]; | |
143 | if (nextState == DFAConst.UNREACHABLE_STATE) { |
|
143 | if (nextState == DFAConst.UNREACHABLE_STATE) { | |
144 | if (m_currentState.final) |
|
144 | if (m_currentState.final) | |
145 | return true; |
|
145 | return true; | |
146 |
|
146 | |||
147 | throw new ParserException( |
|
147 | throw new ParserException( | |
148 | String.Format( |
|
148 | String.Format( | |
149 | "Unexpected symbol '{0}', at pos {1}", |
|
149 | "Unexpected symbol '{0}', at pos {1}", | |
150 | m_buffer[m_pointer], |
|
150 | m_buffer[m_pointer], | |
151 | Position |
|
151 | Position | |
152 | ) |
|
152 | ) | |
153 | ); |
|
153 | ); | |
154 | } |
|
154 | } | |
155 | m_currentState = m_config.states[nextState]; |
|
155 | m_currentState = m_config.states[nextState]; | |
156 | m_tokenLen++; |
|
156 | m_tokenLen++; | |
157 |
|
157 | |||
158 | } while (Shift()); |
|
158 | } while (Shift()); | |
159 |
|
159 | |||
160 | // END OF DATA |
|
160 | // END OF DATA | |
161 | if (!m_currentState.final) |
|
161 | if (!m_currentState.final) | |
162 | throw new ParserException("Unexpected end of data"); |
|
162 | throw new ParserException("Unexpected end of data"); | |
163 |
|
163 | |||
164 | return true; |
|
164 | return true; | |
165 | } |
|
165 | } | |
166 |
|
166 | |||
167 |
|
167 | |||
168 | bool Shift() { |
|
168 | bool Shift() { | |
169 | m_pointer++; |
|
169 | m_pointer++; | |
170 |
|
170 | |||
171 | if (m_pointer >= m_bufferSize) { |
|
171 | if (m_pointer >= m_bufferSize) { | |
172 | if (!ReadNextChunk()) |
|
172 | if (!ReadNextChunk()) | |
173 | return false; |
|
173 | return false; | |
174 | } |
|
174 | } | |
175 |
|
175 | |||
176 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; |
|
176 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; | |
177 |
|
177 | |||
178 | return true; |
|
178 | return true; | |
179 | } |
|
179 | } | |
180 |
|
180 | |||
181 | bool ReadNextChunk() { |
|
181 | bool ReadNextChunk() { | |
182 | if (m_reader == null) |
|
182 | if (m_reader == null) | |
183 | return false; |
|
183 | return false; | |
184 |
|
184 | |||
185 | // extend buffer if nesessary |
|
185 | // extend buffer if nesessary | |
186 | if (m_pointer + m_chunkSize > m_buffer.Length) { |
|
186 | if (m_pointer + m_chunkSize > m_buffer.Length) { | |
187 | // trim unused buffer head |
|
187 | // trim unused buffer head | |
188 | var size = m_tokenLen + m_chunkSize; |
|
188 | var size = m_tokenLen + m_chunkSize; | |
189 | if (size >= m_limit) |
|
189 | if (size >= m_limit) | |
190 | throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit)); |
|
190 | throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit)); | |
191 | var temp = new char[size]; |
|
191 | var temp = new char[size]; | |
192 | Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen); |
|
192 | Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen); | |
193 | m_pointer -= m_tokenOffset; |
|
193 | m_pointer -= m_tokenOffset; | |
194 | m_bufferSize -= m_tokenOffset; |
|
194 | m_bufferSize -= m_tokenOffset; | |
195 | m_tokenOffset = 0; |
|
195 | m_tokenOffset = 0; | |
196 | m_buffer = temp; |
|
196 | m_buffer = temp; | |
197 | } |
|
197 | } | |
198 |
|
198 | |||
199 | var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize); |
|
199 | var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize); | |
200 | if (read == 0) |
|
200 | if (read == 0) | |
201 | return false; |
|
201 | return false; | |
202 |
|
202 | |||
203 | m_bufferSize += read; |
|
203 | m_bufferSize += read; | |
204 |
|
204 | |||
205 | return true; |
|
205 | return true; | |
206 | } |
|
206 | } | |
207 |
|
207 | |||
208 | /// <summary> |
|
208 | /// <summary> | |
209 | /// ΠΠΎΠ·ΠΈΡΠΈΡ ΡΠΊΠ°Π½Π½Π΅ΡΠ° Π²ΠΎ Π²Ρ ΠΎΠ΄Π½ΠΎΠΌ Π±ΡΡΠ΅ΡΠ΅ |
|
209 | /// ΠΠΎΠ·ΠΈΡΠΈΡ ΡΠΊΠ°Π½Π½Π΅ΡΠ° Π²ΠΎ Π²Ρ ΠΎΠ΄Π½ΠΎΠΌ Π±ΡΡΠ΅ΡΠ΅ | |
210 | /// </summary> |
|
210 | /// </summary> | |
211 | public int Position { |
|
211 | public int Position { | |
212 | get { |
|
212 | get { | |
213 | return m_pointer + 1; |
|
213 | return m_pointer + 1; | |
214 | } |
|
214 | } | |
215 | } |
|
215 | } | |
216 |
|
216 | |||
217 | /// <summary> |
|
217 | /// <summary> | |
218 | /// ΠΡΠ΅ΠΊΠ»ΡΡΠ°Π΅Ρ Π²Π½ΡΡΡΠ΅Π½Π½ΠΈΠΉ ΠΠΠ Π½Π° ΡΠΊΠ°Π·Π°Π½Π½ΡΠΉ, ΠΏΠΎΠ·Π²ΠΎΠ»ΡΠ΅Ρ ΡΠ΅Π°Π»ΠΈΠ·ΠΎΠ²Π°ΡΡ ΠΏΠΎΠ΄ΠΎΠ±ΠΈΠ΅ Π·Π°Ρ Π²Π°ΡΡΠ²Π°ΡΡΠ΅ΠΉ |
|
218 | /// ΠΡΠ΅ΠΊΠ»ΡΡΠ°Π΅Ρ Π²Π½ΡΡΡΠ΅Π½Π½ΠΈΠΉ ΠΠΠ Π½Π° ΡΠΊΠ°Π·Π°Π½Π½ΡΠΉ, ΠΏΠΎΠ·Π²ΠΎΠ»ΡΠ΅Ρ ΡΠ΅Π°Π»ΠΈΠ·ΠΎΠ²Π°ΡΡ ΠΏΠΎΠ΄ΠΎΠ±ΠΈΠ΅ Π·Π°Ρ Π²Π°ΡΡΠ²Π°ΡΡΠ΅ΠΉ | |
219 | /// Π³ΡΡΠΏΠΏΠΈΡΠΎΠ²ΠΊΠΈ. |
|
219 | /// Π³ΡΡΠΏΠΏΠΈΡΠΎΠ²ΠΊΠΈ. | |
220 | /// </summary> |
|
220 | /// </summary> | |
221 | /// <param name = "config"></param> |
|
221 | /// <param name = "config"></param> | |
222 | protected void Switch(ScannerConfig config) { |
|
222 | protected void Switch(ScannerConfig config) { | |
223 | Safe.ArgumentNotNull(config.states, "config.states"); |
|
223 | Safe.ArgumentNotNull(config.states, "config.states"); | |
224 |
|
224 | |||
225 | m_defs.Push(m_config); |
|
225 | m_defs.Push(m_config); | |
226 | m_config = config; |
|
226 | m_config = config; | |
227 |
|
227 | |||
228 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; |
|
228 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; | |
229 | } |
|
229 | } | |
230 |
|
230 | |||
231 | /// <summary> |
|
231 | /// <summary> | |
232 | /// ΠΠΎΡΡΡΠ°Π½Π°Π²Π»ΠΈΠ²Π°Π΅Ρ ΠΏΡΠ΅Π΄ΡΠ΄ΡΡΠ΅ΠΉ ΠΠΠ ΡΠΊΠ°Π½Π½Π΅ΡΠ°. |
|
232 | /// ΠΠΎΡΡΡΠ°Π½Π°Π²Π»ΠΈΠ²Π°Π΅Ρ ΠΏΡΠ΅Π΄ΡΠ΄ΡΡΠ΅ΠΉ ΠΠΠ ΡΠΊΠ°Π½Π½Π΅ΡΠ°. | |
233 | /// </summary> |
|
233 | /// </summary> | |
234 | protected void Restore() { |
|
234 | protected void Restore() { | |
235 | if (m_defs.Count == 0) |
|
235 | if (m_defs.Count == 0) | |
236 | throw new InvalidOperationException(); |
|
236 | throw new InvalidOperationException(); | |
237 | m_config = m_defs.Pop(); |
|
237 | m_config = m_defs.Pop(); | |
238 |
|
238 | |||
239 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; |
|
239 | m_previewCode = m_config.alphabet[m_buffer[m_pointer]]; | |
240 | } |
|
240 | } | |
241 |
|
241 | |||
242 | protected override void Dispose(bool disposing) { |
|
242 | protected override void Dispose(bool disposing) { | |
243 | if (disposing) { |
|
243 | if (disposing) { | |
244 | if (m_reader != null && m_disposeReader) |
|
244 | if (m_reader != null && m_disposeReader) | |
245 | m_reader.Dispose(); |
|
245 | m_reader.Dispose(); | |
246 | m_buffer = null; |
|
246 | m_buffer = null; | |
247 | m_bufferSize = 0; |
|
247 | m_bufferSize = 0; | |
248 | m_pointer = 0; |
|
248 | m_pointer = 0; | |
249 | m_tokenLen = 0; |
|
249 | m_tokenLen = 0; | |
250 | m_tokenOffset = 0; |
|
250 | m_tokenOffset = 0; | |
251 | } |
|
251 | } | |
252 | base.Dispose(disposing); |
|
252 | base.Dispose(disposing); | |
253 | } |
|
253 | } | |
254 | } |
|
254 | } | |
255 | } |
|
255 | } |
@@ -1,125 +1,45 | |||||
1 | using System; |
|
1 | using System; | |
2 | using Implab.Automaton.RegularExpressions; |
|
2 | using Implab.Automaton.RegularExpressions; | |
3 | using Implab.Automaton; |
|
3 | using Implab.Automaton; | |
|
4 | using System.Diagnostics; | |||
4 |
|
5 | |||
5 | namespace Implab.Formats { |
|
6 | namespace Implab.Formats { | |
6 | public struct BufferScanner<TTag> { |
|
7 | public struct BufferScanner<TTag> { | |
7 | char[] m_buffer; |
|
|||
8 | int m_offset; |
|
|||
9 | int m_position; |
|
|||
10 | int m_hi; |
|
|||
11 |
|
||||
12 | readonly int m_chunk; |
|
|||
13 | readonly int m_limit; |
|
|||
14 |
|
||||
15 | readonly DFAStateDescriptor<TTag>[] m_dfa; |
|
8 | readonly DFAStateDescriptor<TTag>[] m_dfa; | |
16 | int m_state; |
|
9 | int m_state; | |
|
10 | int m_pos; | |||
17 |
|
11 | |||
18 |
public BufferScanner(DFAStateDescriptor<TTag>[] dfa, int initialState |
|
12 | public BufferScanner(DFAStateDescriptor<TTag>[] dfa, int initialState) { | |
19 | m_dfa = dfa; |
|
13 | m_dfa = dfa; | |
20 | m_state = initialState; |
|
14 | m_state = initialState; | |
21 | m_chunk = chunk; |
|
|||
22 | m_limit = limit; |
|
|||
23 | m_buffer = null; |
|
|||
24 | m_offset = 0; |
|
|||
25 | m_position = 0; |
|
|||
26 | m_hi = 0; |
|
|||
27 | } |
|
|||
28 |
|
||||
29 | public char[] Buffer { |
|
|||
30 | get { |
|
|||
31 | return m_buffer; |
|
|||
32 | } |
|
|||
33 | } |
|
|||
34 |
|
||||
35 | public int HiMark { |
|
|||
36 | get { |
|
|||
37 | return m_hi; |
|
|||
38 | } |
|
|||
39 | } |
|
15 | } | |
40 |
|
16 | |||
41 | public int Position { |
|
17 | public int Position { | |
42 | get { |
|
18 | get { return m_pos; } | |
43 | return m_position; |
|
|||
44 | } |
|
|||
45 | } |
|
|||
46 |
|
||||
47 | public int Length { |
|
|||
48 | get { |
|
|||
49 | return m_hi - m_position; |
|
|||
50 | } |
|
|||
51 | } |
|
|||
52 |
|
||||
53 | public int TokenOffset { |
|
|||
54 | get { |
|
|||
55 | return m_offset; |
|
|||
56 | } |
|
|||
57 | } |
|
|||
58 |
|
||||
59 | public int TokenLength { |
|
|||
60 | get { |
|
|||
61 | return m_position - m_offset; |
|
|||
62 | } |
|
|||
63 | } |
|
|||
64 |
|
||||
65 | public void Init(char[] buffer, int position, int length) { |
|
|||
66 | m_buffer = buffer; |
|
|||
67 | m_position = position; |
|
|||
68 | m_offset = position; |
|
|||
69 | m_hi = position + length; |
|
|||
70 | } |
|
|||
71 |
|
||||
72 | public int Extend() { |
|
|||
73 | // free space |
|
|||
74 | var free = m_buffer.Length - m_hi; |
|
|||
75 |
|
||||
76 | // if the buffer have enough free space |
|
|||
77 | if (free > 0) |
|
|||
78 | return free; |
|
|||
79 |
|
||||
80 | // effective size of the buffer |
|
|||
81 | var size = m_buffer.Length - m_offset; |
|
|||
82 |
|
||||
83 | // calculate the new size |
|
|||
84 | int grow = Math.Min(m_limit - size, m_chunk); |
|
|||
85 | if (grow <= 0) |
|
|||
86 | throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit)); |
|
|||
87 |
|
||||
88 | var temp = new char[size + grow]; |
|
|||
89 | Array.Copy(m_buffer, m_offset, temp, 0, m_hi - m_offset); |
|
|||
90 | m_position -= m_offset; |
|
|||
91 | m_hi -= m_offset; |
|
|||
92 | m_offset = 0; |
|
|||
93 | m_buffer = temp; |
|
|||
94 |
|
||||
95 | return free + grow; |
|
|||
96 | } |
|
|||
97 |
|
||||
98 | public void RaiseMark(int size) { |
|
|||
99 | m_hi += size; |
|
|||
100 | } |
|
19 | } | |
101 |
|
20 | |||
102 | /// <summary> |
|
21 | /// <summary> | |
103 | /// Scan this instance. |
|
22 | /// Scan this instance. | |
104 | /// </summary> |
|
23 | /// </summary> | |
105 | /// <returns><c>true</c> - additional data required</returns> |
|
24 | /// <returns><c>true</c> - additional data required</returns> | |
106 | public bool Scan() { |
|
25 | public bool Scan(int[] buffer, int position, int length) { | |
107 |
|
|
26 | var hi = position + length; | |
108 |
|
|
27 | m_pos = position; | |
109 | var next = m_dfa[m_state].transitions[(int)ch]; |
|
28 | ||
|
29 | while (position < hi) { | |||
|
30 | var next = m_dfa[m_state].transitions[buffer[position]]; | |||
110 | if (next == DFAConst.UNREACHABLE_STATE) { |
|
31 | if (next == DFAConst.UNREACHABLE_STATE) { | |
111 | if (m_dfa[m_state].final) |
|
32 | if (m_dfa[m_state].final) | |
112 | return false; |
|
33 | return false; | |
113 |
|
34 | |||
114 | throw new ParserException( |
|
35 | throw new ParserException( | |
115 | String.Format( |
|
36 | String.Format( | |
116 |
"Unexpected |
|
37 | "Unexpected symbol" | |
117 | new string(m_buffer, m_offset, m_position - m_offset) |
|
|||
118 | ) |
|
38 | ) | |
119 | ); |
|
39 | ); | |
120 | } |
|
40 | } | |
|
41 | m_pos++; | |||
121 | m_state = next; |
|
42 | m_state = next; | |
122 | m_position++; |
|
|||
123 | } |
|
43 | } | |
124 |
|
44 | |||
125 | return true; |
|
45 | return true; | |
@@ -129,8 +49,7 namespace Implab.Formats { | |||||
129 | if (!m_dfa[m_state].final) |
|
49 | if (!m_dfa[m_state].final) | |
130 | throw new ParserException( |
|
50 | throw new ParserException( | |
131 | String.Format( |
|
51 | String.Format( | |
132 |
"Unexpected |
|
52 | "Unexpected EOF" | |
133 | new string(m_buffer, m_offset, m_position - m_offset) |
|
|||
134 | ) |
|
53 | ) | |
135 | ); |
|
54 | ); | |
136 | } |
|
55 | } |
@@ -4,11 +4,11 using Implab.Components; | |||||
4 | namespace Implab.Formats { |
|
4 | namespace Implab.Formats { | |
5 | public abstract class TextScanner<TTag> : Disposable { |
|
5 | public abstract class TextScanner<TTag> : Disposable { | |
6 |
|
6 | |||
7 |
|
|
7 | readonly int[] m_buffer; | |
8 |
int m_ |
|
8 | int m_bufferOffset; | |
9 |
int m_ |
|
9 | int m_dataLength; | |
10 | int m_tokenOffset; |
|
|||
11 | int m_tokenLength; |
|
10 | int m_tokenLength; | |
|
11 | ||||
12 | TTag[] m_tags; |
|
12 | TTag[] m_tags; | |
13 |
|
13 | |||
14 | BufferScanner<TTag> m_scanner; |
|
14 | BufferScanner<TTag> m_scanner; | |
@@ -17,29 +17,36 namespace Implab.Formats { | |||||
17 | if (EOF) |
|
17 | if (EOF) | |
18 | return false; |
|
18 | return false; | |
19 |
|
19 | |||
20 |
// create a new scanner from template (scanners are |
|
20 | // create a new scanner from template (scanners are value types) | |
21 | var inst = m_scanner; |
|
21 | var inst = m_scanner; | |
22 |
|
22 | |||
23 | // initialize the scanner |
|
23 | m_tokenLength = 0; | |
24 | inst.Init(m_buffer, m_offset, m_length); |
|
24 | ||
|
25 | while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) { | |||
|
26 | m_tokenLength += m_dataLength; | |||
|
27 | ||||
|
28 | var actual = Read(m_buffer, 0, m_buffer.Length); | |||
|
29 | ||||
|
30 | m_bufferOffset = 0; | |||
|
31 | m_dataLength = actual; | |||
25 |
|
32 | |||
26 | // do work |
|
33 | if (actual == 0) { | |
27 |
|
|
34 | inst.Eof(); | |
28 |
|
|
35 | break; | |
|
36 | } | |||
|
37 | } | |||
|
38 | ||||
|
39 | var len = inst.Position - m_bufferOffset; | |||
|
40 | m_tokenLength += len; | |||
|
41 | m_dataLength -= len; | |||
|
42 | m_bufferOffset = inst.Position; | |||
29 |
|
43 | |||
30 | // save result; |
|
44 | // save result; | |
31 | m_buffer = inst.Buffer; |
|
45 | ||
32 | m_length = inst.Length; |
|
|||
33 | m_offset = inst.Position; |
|
|||
34 | m_tokenOffset = inst.TokenOffset; |
|
|||
35 | m_tokenLength = inst.TokenLength; |
|
|||
36 |
|
||||
37 | m_tags = inst.GetTokenTags(); |
|
46 | m_tags = inst.GetTokenTags(); | |
38 | } |
|
47 | } | |
39 |
|
48 | |||
40 | protected string GetToken() { |
|
49 | protected abstract int Read(int[] buffer, int offset, int size); | |
41 | return new String(m_buffer, m_tokenOffset, m_tokenLength); |
|
|||
42 | } |
|
|||
43 |
|
50 | |||
44 | protected TTag[] Tags { |
|
51 | protected TTag[] Tags { | |
45 | get { |
|
52 | get { | |
@@ -47,26 +54,8 namespace Implab.Formats { | |||||
47 | } |
|
54 | } | |
48 | } |
|
55 | } | |
49 |
|
56 | |||
50 | /// <summary> |
|
|||
51 | /// Feed the specified scanner. |
|
|||
52 | /// </summary> |
|
|||
53 | /// <param name="scanner">Scanner.</param> |
|
|||
54 | /// <example> |
|
|||
55 | /// protected override void Feed(ref BufferScanner<TTag> scanner) { |
|
|||
56 | /// var size = scanner.Extend(); |
|
|||
57 | /// var actual = m_reader.Read(scanner.Buffer, scanner.HiMark, size); |
|
|||
58 | /// if (actual == 0) { |
|
|||
59 | /// m_eof = true; |
|
|||
60 | /// scanner.Eof(); |
|
|||
61 | /// } else { |
|
|||
62 | /// scanner.RaiseHiMark(actual); |
|
|||
63 | /// } |
|
|||
64 | /// } |
|
|||
65 | /// </example> |
|
|||
66 | protected abstract void Feed(ref BufferScanner<TTag> scanner); |
|
|||
67 |
|
||||
68 | public abstract bool EOF { get; } |
|
57 | public abstract bool EOF { get; } | |
69 |
|
58 | |||
70 | } |
|
59 | } | |
71 | } |
|
60 | } | |
72 |
|
61 |
General Comments 0
You need to be logged in to leave comments.
Login now