##// END OF EJS Templates
implab » ImplabNet
RSS
sync
cin -
r174:983df35b3ca1 ref20160224
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,125 +1,45
1 using System;
1 using System;
2 using Implab.Automaton.RegularExpressions;
2 using Implab.Automaton.RegularExpressions;
3 using Implab.Automaton;
3 using Implab.Automaton;
4 using System.Diagnostics;
4
5
5 namespace Implab.Formats {
6 namespace Implab.Formats {
6 public struct BufferScanner<TTag> {
7 public struct BufferScanner<TTag> {
7 char[] m_buffer;
8 int m_offset;
9 int m_position;
10 int m_hi;
11
12 readonly int m_chunk;
13 readonly int m_limit;
14
15 readonly DFAStateDescriptor<TTag>[] m_dfa;
8 readonly DFAStateDescriptor<TTag>[] m_dfa;
16 int m_state;
9 int m_state;
10 int m_pos;
17
11
18 public BufferScanner(DFAStateDescriptor<TTag>[] dfa, int initialState, int chunk, int limit) {
12 public BufferScanner(DFAStateDescriptor<TTag>[] dfa, int initialState) {
19 m_dfa = dfa;
13 m_dfa = dfa;
20 m_state = initialState;
14 m_state = initialState;
21 m_chunk = chunk;
22 m_limit = limit;
23 m_buffer = null;
24 m_offset = 0;
25 m_position = 0;
26 m_hi = 0;
27 }
28
29 public char[] Buffer {
30 get {
31 return m_buffer;
32 }
33 }
34
35 public int HiMark {
36 get {
37 return m_hi;
38 }
39 }
15 }
40
16
41 public int Position {
17 public int Position {
42 get {
18 get { return m_pos; }
43 return m_position;
44 }
45 }
46
47 public int Length {
48 get {
49 return m_hi - m_position;
50 }
51 }
52
53 public int TokenOffset {
54 get {
55 return m_offset;
56 }
57 }
58
59 public int TokenLength {
60 get {
61 return m_position - m_offset;
62 }
63 }
64
65 public void Init(char[] buffer, int position, int length) {
66 m_buffer = buffer;
67 m_position = position;
68 m_offset = position;
69 m_hi = position + length;
70 }
71
72 public int Extend() {
73 // free space
74 var free = m_buffer.Length - m_hi;
75
76 // if the buffer have enough free space
77 if (free > 0)
78 return free;
79
80 // effective size of the buffer
81 var size = m_buffer.Length - m_offset;
82
83 // calculate the new size
84 int grow = Math.Min(m_limit - size, m_chunk);
85 if (grow <= 0)
86 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
87
88 var temp = new char[size + grow];
89 Array.Copy(m_buffer, m_offset, temp, 0, m_hi - m_offset);
90 m_position -= m_offset;
91 m_hi -= m_offset;
92 m_offset = 0;
93 m_buffer = temp;
94
95 return free + grow;
96 }
97
98 public void RaiseMark(int size) {
99 m_hi += size;
100 }
19 }
101
20
102 /// <summary>
21 /// <summary>
103 /// Scan this instance.
22 /// Scan this instance.
104 /// </summary>
23 /// </summary>
105 /// <returns><c>true</c> - additional data required</returns>
24 /// <returns><c>true</c> - additional data required</returns>
106 public bool Scan() {
25 public bool Scan(int[] buffer, int position, int length) {
107 while (m_position < m_hi) {
26 var hi = position + length;
108 var ch = m_buffer[m_position];
27 m_pos = position;
109 var next = m_dfa[m_state].transitions[(int)ch];
28
29 while (position < hi) {
30 var next = m_dfa[m_state].transitions[buffer[position]];
110 if (next == DFAConst.UNREACHABLE_STATE) {
31 if (next == DFAConst.UNREACHABLE_STATE) {
111 if (m_dfa[m_state].final)
32 if (m_dfa[m_state].final)
112 return false;
33 return false;
113
34
114 throw new ParserException(
35 throw new ParserException(
115 String.Format(
36 String.Format(
116 "Unexpected token '{0}'",
37 "Unexpected symbol"
117 new string(m_buffer, m_offset, m_position - m_offset)
118 )
38 )
119 );
39 );
120 }
40 }
41 m_pos++;
121 m_state = next;
42 m_state = next;
122 m_position++;
123 }
43 }
124
44
125 return true;
45 return true;
@@ -129,8 +49,7 namespace Implab.Formats {
129 if (!m_dfa[m_state].final)
49 if (!m_dfa[m_state].final)
130 throw new ParserException(
50 throw new ParserException(
131 String.Format(
51 String.Format(
132 "Unexpected token '{0}'",
52 "Unexpected EOF"
133 new string(m_buffer, m_offset, m_position - m_offset)
134 )
53 )
135 );
54 );
136 }
55 }
Show file before | Show file after | Hide comments
@@ -4,11 +4,11 using Implab.Components;
4 namespace Implab.Formats {
4 namespace Implab.Formats {
5 public abstract class TextScanner<TTag> : Disposable {
5 public abstract class TextScanner<TTag> : Disposable {
6
6
7 char[] m_buffer;
7 readonly int[] m_buffer;
8 int m_offset;
8 int m_bufferOffset;
9 int m_length;
9 int m_dataLength;
10 int m_tokenOffset;
11 int m_tokenLength;
10 int m_tokenLength;
11
12 TTag[] m_tags;
12 TTag[] m_tags;
13
13
14 BufferScanner<TTag> m_scanner;
14 BufferScanner<TTag> m_scanner;
@@ -17,29 +17,36 namespace Implab.Formats {
17 if (EOF)
17 if (EOF)
18 return false;
18 return false;
19
19
20 // create a new scanner from template (scanners are structs)
20 // create a new scanner from template (scanners are value types)
21 var inst = m_scanner;
21 var inst = m_scanner;
22
22
23 // initialize the scanner
23 m_tokenLength = 0;
24 inst.Init(m_buffer, m_offset, m_length);
24
25 while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {
26 m_tokenLength += m_dataLength;
27
28 var actual = Read(m_buffer, 0, m_buffer.Length);
29
30 m_bufferOffset = 0;
31 m_dataLength = actual;
25
32
26 // do work
33 if (actual == 0) {
27 while (inst.Scan())
34 inst.Eof();
28 Feed(ref inst);
35 break;
36 }
37 }
38
39 var len = inst.Position - m_bufferOffset;
40 m_tokenLength += len;
41 m_dataLength -= len;
42 m_bufferOffset = inst.Position;
29
43
30 // save result;
44 // save result;
31 m_buffer = inst.Buffer;
32 m_length = inst.Length;
33 m_offset = inst.Position;
34 m_tokenOffset = inst.TokenOffset;
35 m_tokenLength = inst.TokenLength;
36
45
37 m_tags = inst.GetTokenTags();
46 m_tags = inst.GetTokenTags();
38 }
47 }
39
48
40 protected string GetToken() {
49 protected abstract int Read(int[] buffer, int offset, int size);
41 return new String(m_buffer, m_tokenOffset, m_tokenLength);
42 }
43
50
44 protected TTag[] Tags {
51 protected TTag[] Tags {
45 get {
52 get {
@@ -47,24 +54,6 namespace Implab.Formats {
47 }
54 }
48 }
55 }
49
56
50 /// <summary>
51 /// Feed the specified scanner.
52 /// </summary>
53 /// <param name="scanner">Scanner.</param>
54 /// <example>
55 /// protected override void Feed(ref BufferScanner<TTag> scanner) {
56 /// var size = scanner.Extend();
57 /// var actual = m_reader.Read(scanner.Buffer, scanner.HiMark, size);
58 /// if (actual == 0) {
59 /// m_eof = true;
60 /// scanner.Eof();
61 /// } else {
62 /// scanner.RaiseHiMark(actual);
63 /// }
64 /// }
65 /// </example>
66 protected abstract void Feed(ref BufferScanner<TTag> scanner);
67
68 public abstract bool EOF { get; }
57 public abstract bool EOF { get; }
69
58
70 }
59 }
General Comments 0
You need to be logged in to leave comments. Login now