implab/ImplabNet Files · Implab/Formats/TextScanner.cs

sync

cin - - Load All Authors

File last commit:

r174:983df35b3ca1 ref20160224


                r174:983df35b3ca1

ref20160224

Download file

             TextScanner.cs
        
                    61 lines
            
             | 1.4 KiB
            
                | text/x-csharp
            
             |
                CSharpLexer
            
             / Implab / Formats / TextScanner.cs
          
                    History
                
                 |
                  Annotation
                 | Raw
                 |Copy content
                 |Copy permalink

      using System;

      using Implab.Components;

      namespace Implab.Formats {

          public abstract class TextScanner<TTag> : Disposable {

              readonly int[] m_buffer;

              int m_bufferOffset;

              int m_dataLength;

              int m_tokenLength;

              TTag[] m_tags;

              BufferScanner<TTag> m_scanner;

              protected bool ReadTokenInternal() {

                  if (EOF)

                      return false;

                  // create a new scanner from template (scanners are value types)

                  var inst = m_scanner;

                  m_tokenLength = 0;

                  while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {

                      m_tokenLength += m_dataLength;

                      var actual = Read(m_buffer, 0, m_buffer.Length);

                      m_bufferOffset = 0;

                      m_dataLength = actual;

                      if (actual == 0) {

                          inst.Eof();

                          break;

                      }

                  }

                  var len = inst.Position - m_bufferOffset;

                  m_tokenLength += len;

                  m_dataLength -= len;

                  m_bufferOffset = inst.Position;

                  // save result;

                  m_tags = inst.GetTokenTags();

              }

              protected abstract int Read(int[] buffer, int offset, int size);

              protected TTag[] Tags {

                  get {

                      return m_tags; 

                  }

              }

              public abstract bool EOF { get; }

          }

      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

				using System;
				using Implab.Components;

				namespace Implab.Formats {
				public abstract class TextScanner<TTag> : Disposable {

				readonly int[] m_buffer;
				int m_bufferOffset;
				int m_dataLength;
				int m_tokenLength;

				TTag[] m_tags;

				BufferScanner<TTag> m_scanner;

				protected bool ReadTokenInternal() {
				if (EOF)
				return false;

				// create a new scanner from template (scanners are value types)
				var inst = m_scanner;

				m_tokenLength = 0;

				while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {
				m_tokenLength += m_dataLength;

				var actual = Read(m_buffer, 0, m_buffer.Length);

				m_bufferOffset = 0;
				m_dataLength = actual;

				if (actual == 0) {
				inst.Eof();
				break;
				}
				}

				var len = inst.Position - m_bufferOffset;
				m_tokenLength += len;
				m_dataLength -= len;
				m_bufferOffset = inst.Position;

				// save result;

				m_tags = inst.GetTokenTags();
				}

				protected abstract int Read(int[] buffer, int offset, int size);

				protected TTag[] Tags {
				get {
				return m_tags;
				}
				}

				public abstract bool EOF { get; }

				}
				}