implab/ImplabNet Files · Implab/Formats/TextScanner.cs

sync

cin - - Load All Authors

File last commit:

r174:983df35b3ca1 ref20160224


                r174:983df35b3ca1

ref20160224

Download file

             TextScanner.cs
        
                    61 lines
            
             | 1.4 KiB
            
                | text/x-csharp
            
             |
                CSharpLexer
            
             / Implab / Formats / TextScanner.cs
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        cin
    
Working on text scanner

              r173
            
      using System;

      using Implab.Components;

      namespace Implab.Formats {

          public abstract class TextScanner<TTag> : Disposable {

        cin
    
sync

              r174
            
              readonly int[] m_buffer;

              int m_bufferOffset;

              int m_dataLength;

        cin
    
Working on text scanner

              r173
            
              int m_tokenLength;

        cin
    
sync

              r174
            
        cin
    
Working on text scanner

              r173
            
              TTag[] m_tags;

              BufferScanner<TTag> m_scanner;

              protected bool ReadTokenInternal() {

                  if (EOF)

                      return false;

        cin
    
sync

              r174
            
                  // create a new scanner from template (scanners are value types)

        cin
    
Working on text scanner

              r173
            
                  var inst = m_scanner;

        cin
    
sync

              r174
            
                  m_tokenLength = 0;

                  while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {

                      m_tokenLength += m_dataLength;

                      var actual = Read(m_buffer, 0, m_buffer.Length);

                      m_bufferOffset = 0;

                      m_dataLength = actual;

        cin
    
Working on text scanner

              r173
            
        cin
    
sync

              r174
            
                      if (actual == 0) {

                          inst.Eof();

                          break;

                      }

                  }

                  var len = inst.Position - m_bufferOffset;

                  m_tokenLength += len;

                  m_dataLength -= len;

                  m_bufferOffset = inst.Position;

        cin
    
Working on text scanner

              r173
            
                  // save result;

        cin
    
sync

              r174
            
        cin
    
Working on text scanner

              r173
            
                  m_tags = inst.GetTokenTags();

              }

        cin
    
sync

              r174
            
              protected abstract int Read(int[] buffer, int offset, int size);

        cin
    
Working on text scanner

              r173
            
              protected TTag[] Tags {

                  get {

                      return m_tags; 

                  }

              }

              public abstract bool EOF { get; }

        cin
    
sync

              r174
            
        cin
    
Working on text scanner

              r173
            
          }

      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

cin Working on text scanner	r173	using System;
		using Implab.Components;

		namespace Implab.Formats {
		public abstract class TextScanner<TTag> : Disposable {

cin sync	r174	readonly int[] m_buffer;
		int m_bufferOffset;
		int m_dataLength;
cin Working on text scanner	r173	int m_tokenLength;
cin sync	r174
cin Working on text scanner	r173	TTag[] m_tags;

		BufferScanner<TTag> m_scanner;

		protected bool ReadTokenInternal() {
		if (EOF)
		return false;

cin sync	r174	// create a new scanner from template (scanners are value types)
cin Working on text scanner	r173	var inst = m_scanner;

cin sync	r174	m_tokenLength = 0;

		while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {
		m_tokenLength += m_dataLength;

		var actual = Read(m_buffer, 0, m_buffer.Length);

		m_bufferOffset = 0;
		m_dataLength = actual;
cin Working on text scanner	r173
cin sync	r174	if (actual == 0) {
		inst.Eof();
		break;
		}
		}

		var len = inst.Position - m_bufferOffset;
		m_tokenLength += len;
		m_dataLength -= len;
		m_bufferOffset = inst.Position;
cin Working on text scanner	r173
		// save result;
cin sync	r174
cin Working on text scanner	r173	m_tags = inst.GetTokenTags();
		}

cin sync	r174	protected abstract int Read(int[] buffer, int offset, int size);
cin Working on text scanner	r173
		protected TTag[] Tags {
		get {
		return m_tags;
		}
		}

		public abstract bool EOF { get; }
cin sync	r174
cin Working on text scanner	r173	}
		}