implab/ImplabNet Commit - r60:10c7337d29e7

+JSONXmlReaderOptions...

cin -

r60:10c7337d29e7 default

parent child

Context file:

r60:10c7337d29e7

Collapse all files

Implab/JSON/JSONXmlReaderOptions.cs created 644 +61 0

@@ -0,0 +1,61
	1	using System;
	2	using System.Collections.Generic;
	3	using System.Linq;
	4	using System.Text;
	5	using System.Xml;
	6
	7	namespace Implab.JSON {
	8	/// <summary>
	9	/// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
	10	/// интерпретации <c>JSON</c> документа.
	11	/// </summary>
	12	public class JSONXmlReaderOptions {
	13	/// <summary>
	14	/// Пространство имен в котором будут располагаться читаемые элементы документа
	15	/// </summary>
	16	public string NamespaceURI {
	17	get;
	18	set;
	19	}
	20
	21	/// <summary>
	22	/// Интерпретировать массивы как множественные элементы (убирает один уровень вложенности)
	23	/// </summary>
	24	public bool FlattenArrays {
	25	get;
	26	set;
	27	}
	28
	29	/// <summary>
	30	/// Префикс, для узлов документа
	31	/// </summary>
	32	public string NodesPrefix {
	33	get;
	34	set;
	35	}
	36
	37	/// <summary>
	38	/// Имя корневого элемента в xml документе
	39	/// </summary>
	40	public string RootName {
	41	get;
	42	set;
	43	}
	44
	45	/// <summary>
	46	/// Имя элемента для массивов, если не включена опция <see cref="FlattenArrays"/>.
	47	/// </summary>
	48	public string ArrayItemName {
	49	get;
	50	set;
	51	}
	52
	53	/// <summary>
	54	/// Таблица атомизированных строк для построения документа.
	55	/// </summary>
	56	public XmlNameTable NameTable {
	57	get;
	58	set;
	59	}
	60	}
	61	}

Implab/Implab.csproj +1 0

             <?xml version="1.0" encoding="utf-8"?>
             <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
               <PropertyGroup>
                 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
                 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
                 <ProductVersion>10.0.0</ProductVersion>
                 <SchemaVersion>2.0</SchemaVersion>
                 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
                 <OutputType>Library</OutputType>
                 <RootNamespace>Implab</RootNamespace>
                 <AssemblyName>Implab</AssemblyName>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>TRACE;DEBUG;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
                 <RunCodeAnalysis>true</RunCodeAnalysis>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
                 <DebugType>full</DebugType>
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <ItemGroup>
                 <Reference Include="System" />
                 <Reference Include="System.XML" />
               </ItemGroup>
               <ItemGroup>
                 <Compile Include="Component.cs" />
                 <Compile Include="CustomEqualityComparer.cs" />
                 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
                 <Compile Include="Diagnostics\EventText.cs" />
                 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
                 <Compile Include="Diagnostics\LogChannel.cs" />
                 <Compile Include="Diagnostics\LogicalOperation.cs" />
                 <Compile Include="Diagnostics\TextFileListener.cs" />
                 <Compile Include="Diagnostics\TextListenerBase.cs" />
                 <Compile Include="Diagnostics\TraceLog.cs" />
                 <Compile Include="Diagnostics\TraceContext.cs" />
                 <Compile Include="Diagnostics\TraceEvent.cs" />
                 <Compile Include="Diagnostics\TraceEventType.cs" />
                 <Compile Include="Disposable.cs" />
                 <Compile Include="ICancellable.cs" />
                 <Compile Include="IProgressHandler.cs" />
                 <Compile Include="IProgressNotifier.cs" />
                 <Compile Include="IPromise.cs" />
                 <Compile Include="IPromiseBase.cs" />
                 <Compile Include="IServiceLocator.cs" />
                 <Compile Include="ITaskController.cs" />
                 <Compile Include="JSON\JSONElementContext.cs" />
                 <Compile Include="JSON\JSONElementType.cs" />
                 <Compile Include="JSON\JSONGrammar.cs" />
                 <Compile Include="JSON\JSONParser.cs" />
                 <Compile Include="JSON\JSONScanner.cs" />
                 <Compile Include="JSON\JsonTokenType.cs" />
                 <Compile Include="JSON\JSONWriter.cs" />
                 <Compile Include="JSON\JSONXmlReader.cs" />
+                <Compile Include="JSON\JSONXmlReaderOptions.cs" />
                 <Compile Include="JSON\StringTranslator.cs" />
                 <Compile Include="Parallels\DispatchPool.cs" />
                 <Compile Include="Parallels\ArrayTraits.cs" />
                 <Compile Include="Parallels\MTQueue.cs" />
                 <Compile Include="Parallels\WorkerPool.cs" />
                 <Compile Include="Parsing\Alphabet.cs" />
                 <Compile Include="Parsing\AlphabetBase.cs" />
                 <Compile Include="Parsing\AltToken.cs" />
                 <Compile Include="Parsing\BinaryToken.cs" />
                 <Compile Include="Parsing\CatToken.cs" />
                 <Compile Include="Parsing\CDFADefinition.cs" />
                 <Compile Include="Parsing\DFABuilder.cs" />
                 <Compile Include="Parsing\DFADefinitionBase.cs" />
                 <Compile Include="Parsing\DFAStateDescriptor.cs" />
                 <Compile Include="Parsing\DFAutomaton.cs" />
                 <Compile Include="Parsing\EDFADefinition.cs" />
                 <Compile Include="Parsing\EmptyToken.cs" />
                 <Compile Include="Parsing\EndToken.cs" />
                 <Compile Include="Parsing\EnumAlphabet.cs" />
                 <Compile Include="Parsing\Grammar.cs" />
                 <Compile Include="Parsing\IAlphabet.cs" />
                 <Compile Include="Parsing\IDFADefinition.cs" />
                 <Compile Include="Parsing\IVisitor.cs" />
                 <Compile Include="Parsing\ParserException.cs" />
                 <Compile Include="Parsing\Scanner.cs" />
                 <Compile Include="Parsing\StarToken.cs" />
                 <Compile Include="Parsing\SymbolToken.cs" />
                 <Compile Include="Parsing\Token.cs" />
                 <Compile Include="ServiceLocator.cs" />
                 <Compile Include="TaskController.cs" />
                 <Compile Include="ProgressInitEventArgs.cs" />
                 <Compile Include="Properties\AssemblyInfo.cs" />
                 <Compile Include="Promise.cs" />
                 <Compile Include="Parallels\AsyncPool.cs" />
                 <Compile Include="Safe.cs" />
                 <Compile Include="ValueEventArgs.cs" />
               </ItemGroup>
               <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
               <ItemGroup />
             </Project>

Implab/JSON/JSONScanner.cs +12 -1

             using Implab.Parsing;
             using System;
             using System.Collections.Generic;
             using System.Globalization;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.JSON {
                 /// <summary>
-                /// Сканнер, разбивающий поток символов на токены JSON.
+                /// Сканнер (лексер), разбивающий поток символов на токены JSON.
                 /// </summary>
                 public class JSONScanner : Scanner {
                     char[] m_stringBuffer;
                     DFAStateDescriptior[] m_stringDFA;
                     int[] m_stringAlphabet;
+                    /// <summary>
+                    /// Создает новый экземпляр сканнера
+                    /// </summary>
                     public JSONScanner()
                         : base(JSONGrammar.Instance.JsonDFA) {
                         m_stringBuffer = new char[1024];
                         var dfa = JSONGrammar.Instance.JsonStringDFA;
                         m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
                         m_stringDFA = dfa.States;
                     }
+                    /// <summary>
+                    /// Читает следующий лексический элемент из входных данных.
+                    /// </summary>
+                    /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
+                    /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
+                    /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
+                    /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
+                    /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
                     public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
                         if (ReadTokenInternal()) {
                             switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
                                     tokenValue = ReadString();
                                     tokenType = JsonTokenType.String;
                                     break;
                                 case JSONGrammar.TokenType.Number:
                                     tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
                                     tokenType = JsonTokenType.Number;
                                     break;
                                 default:
                                     tokenType = (JsonTokenType)m_currentState.tag[0];
                                     tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
                                     break;
                             }
                             return true;
                         }
                         tokenValue = null;
                         tokenType = JsonTokenType.None;
                         return false;
                     }
                     string ReadString() {
                         int pos = 0;
                         Switch(m_stringDFA, m_stringAlphabet);
                         while (ReadTokenInternal()) {
                             switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
                                 case JSONGrammar.TokenType.StringBound:
                                     Restore();
                                     return new String(m_stringBuffer, 0, pos);
                                 case JSONGrammar.TokenType.UnescapedChar:
                                     EnsureStringBufferSize(pos + m_tokenLen);
                                     Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
                                     pos += m_tokenLen;
                                     break;
                                 case JSONGrammar.TokenType.EscapedUnicode:
                                     EnsureStringBufferSize(pos + 1);
                                     m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
                                     pos++;
                                     break;
                                 case JSONGrammar.TokenType.EscapedChar:
                                     EnsureStringBufferSize(pos + 1);
                                     m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
                                     pos++;
                                     break;
                                 default:
                                     break;
                             }
                         }
                         throw new ParserException("Unexpected end of data");
                     }
                     void EnsureStringBufferSize(int size) {
                         if (size > m_stringBuffer.Length) {
                             var newBuffer = new char[size];
                             m_stringBuffer.CopyTo(newBuffer, 0);
                             m_stringBuffer = newBuffer;
                         }
                     }
                 }
             }

Implab/JSON/JSONXmlReader.cs +33 -9

             using Implab;
-            using Implab.JSON;
             using Implab.Parsing;
             using System;
             using System.Collections.Generic;
+            using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             using System.Xml;
-            namespace ConsPlay {
+            namespace Implab.JSON {
                 public class JSONXmlReader : XmlReader {
                     enum ValueContext {
                         Undefined,
                         ElementStart,
                         ElementValue,
                         ElementEnd,
                         ElementEmpty
                     }
                     struct LocalNameContext {
                         public string localName;
                         public bool isArray;
                     }
                     JSONParser m_parser;
                     ValueContext m_valueContext;
                     ReadState m_state = ReadState.Initial;
                     Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
                     LocalNameContext m_localName;
-                    string m_rootName = "json";
-                    string m_prefix = String.Empty;
-                    string m_namespaceUri = String.Empty;
-                    bool m_flattenArrays = false;
-                    NameTable m_nameTable = new NameTable();
                     int m_depthCorrection = 0;
-                    public JSONXmlReader(JSONParser parser) {
+                    readonly string m_rootName;
+                    readonly string m_prefix;
+                    readonly string m_namespaceUri;
+                    readonly bool m_flattenArrays;
+                    readonly string m_arrayItemName;
+                    readonly XmlNameTable m_nameTable;
+                    public JSONXmlReader(JSONParser parser, JSONXmlReaderOptions options) {
                         Safe.ArgumentNotNull(parser, "parser");
                         m_parser = parser;
+                        if (options != null) {
+                            m_prefix = options.NodesPrefix ?? String.Empty;
+                            m_namespaceUri = options.NamespaceURI ?? String.Empty;
+                            m_rootName = options.RootName ?? "json";
+                            m_flattenArrays = options.FlattenArrays;
+                            m_arrayItemName = options.ArrayItemName ?? "item";
+                            m_nameTable = options.NameTable ?? new NameTable();
+                        } else {
+                            m_prefix = String.Empty;
+                            m_namespaceUri = String.Empty;
+                            m_rootName = "json";
+                            m_flattenArrays = false;
+                            m_arrayItemName = "item";
+                            m_nameTable = new NameTable();
+                        }
                     }
                     /// <summary>
                     /// Always 0, JSON doesn't support attributes
                     /// </summary>
                     public override int AttributeCount {
                         get { return 0; }
                     }
                     public override string BaseURI {
                         get { return String.Empty; }
                     }
                     public override int Depth {
                         get {
                             return m_localNameStack.Count+m_depthCorrection;
                         }
                     }
                     public override bool EOF {
                         get { return m_parser.EOF; }
                     }
                     /// <summary>
                     /// Always throws an exception
                     /// </summary>
                     /// <param name="i"></param>
                     /// <returns></returns>
                     public override string GetAttribute(int i) {
                         throw new ArgumentOutOfRangeException();
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <param name="namespaceURI"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name, string namespaceURI) {
                         return String.Empty;
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name) {
                         return String.Empty;
                     }
                     public override bool IsEmptyElement {
                         get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
                     }
                     public override string LocalName {
                         get { return m_localName.localName; }
                     }
                     public override string LookupNamespace(string prefix) {
                         if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
                             return m_namespaceUri;
                         else
                             return String.Empty;
                     }
                     public override bool MoveToAttribute(string name, string ns) {
                         return false;
                     }
                     public override bool MoveToAttribute(string name) {
                         return false;
                     }
                     public override bool MoveToElement() {
                         return false;
                     }
                     public override bool MoveToFirstAttribute() {
                         return false;
                     }
                     public override bool MoveToNextAttribute() {
                         return false;
                     }
                     public override XmlNameTable NameTable {
                         get { return m_nameTable; }
                     }
                     public override string NamespaceURI {
                         get { return m_namespaceUri; }
                     }
                     public override XmlNodeType NodeType {
                         get {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.BeginObject:
                                 case JSONElementType.BeginArray:
                                     return XmlNodeType.Element;
                                 case JSONElementType.EndObject:
                                 case JSONElementType.EndArray:
                                     return XmlNodeType.EndElement;
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                         case ValueContext.ElementEmpty:
                                             return XmlNodeType.Element;
                                         case ValueContext.ElementValue:
                                             return XmlNodeType.Text;
                                         case ValueContext.ElementEnd:
                                             return XmlNodeType.EndElement;
                                         default:
                                             throw new InvalidOperationException();
                                     }
                                 default:
                                     throw new InvalidOperationException();
                             }
                         }
                     }
                     public override string Prefix {
                         get { return m_prefix; }
                     }
                     public override bool Read() {
                         if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
                             return false;
                         if (m_state == ReadState.Initial)
                             m_state = System.Xml.ReadState.Interactive;
                         try {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                             SetLocalName(String.Empty);
                                             m_valueContext = ValueContext.ElementValue;
                                             return true;
                                         case ValueContext.ElementValue:
                                             RestoreLocalName();
                                             m_valueContext = ValueContext.ElementEnd;
                                             return true;
                                         case ValueContext.ElementEmpty:
                                         case ValueContext.ElementEnd:
                                             RestoreLocalName();
                                             break;
                                     }
                                     break;
                                 case JSONElementType.EndArray:
                                 case JSONElementType.EndObject:
                                     RestoreLocalName();
                                     break;
                             }
-                            string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
+                            string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : m_arrayItemName;
                             while (m_parser.Read()) {
                                 if (!String.IsNullOrEmpty(m_parser.ElementName))
                                     itemName = m_parser.ElementName;
                                 switch (m_parser.ElementType) {
                                     case JSONElementType.BeginArray:
                                         if (m_flattenArrays && !m_localName.isArray) {
                                             m_depthCorrection--;
                                             SetLocalName(itemName, true);
                                             continue;
                                         } else {
                                             SetLocalName(itemName, true);
                                         }
                                         break;
                                     case JSONElementType.BeginObject:
                                         SetLocalName(itemName);
                                         break;
                                     case JSONElementType.EndArray:
                                         if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
                                             RestoreLocalName();
                                             m_depthCorrection++;
                                             continue;
                                         }
                                         break;
                                     case JSONElementType.EndObject:
                                         break;
                                     case JSONElementType.Value:
                                         SetLocalName(itemName);
                                         m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
                                         break;
                                     default:
                                         break;
                                 }
                                 return true;
                             }
                             m_state = System.Xml.ReadState.EndOfFile;
                             return false;
                         } catch {
                             m_state = System.Xml.ReadState.Error;
                             throw;
                         }
                     }
                     public override bool ReadAttributeValue() {
                         return false;
                     }
                     public override ReadState ReadState {
                         get { return m_state; }
                     }
                     public override void ResolveEntity() {
                         // do nothing
                     }
                     public override string Value {
                         get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
                     }
                     void SetLocalName(string name) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = false;
                     }
                     void SetLocalName(string name, bool isArray) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = isArray;
                     }
                     void RestoreLocalName() {
                         m_localName = m_localNameStack.Pop();
                     }
                     public override void Close() {
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing) {
                             m_parser.Dispose();
                         }
                         base.Dispose(disposing);
                     }
+                    public static JSONXmlReader OpenFile(string file, JSONXmlReaderOptions options) {
+                        var stream = File.OpenText(file);
+                        var parser = new JSONParser(stream, true);
+                        return new JSONXmlReader(parser, options);
+                    }
                 }
             }

Implab/Parsing/Scanner.cs +2 -1

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 /// <summary>
                 /// Базовый класс для разбора потока входных символов на токены.
                 /// </summary>
                 /// <remarks>
                 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
                 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
                 /// конца токена и допустимости текущего символа.
                 /// </remarks>
                 public abstract class Scanner : Disposable {
                     struct ScannerConfig {
                         public DFAStateDescriptior[] states;
                         public int[] alphabetMap;
                     }
                     Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
                     DFAStateDescriptior[] m_states;
                     int[] m_alphabetMap;
                     protected DFAStateDescriptior m_currentState;
                     int m_previewCode;
                     protected int m_tokenLen = 0;
                     protected int m_tokenOffset;
                     protected char[] m_buffer;
                     protected int m_bufferSize;
                     protected int m_pointer;
                     TextReader m_reader;
                     bool m_disposeReader;
                     int m_chunkSize = 1024; // 1k
                     int m_limit = 10 * 1024 * 1024; // 10Mb
                     public Scanner(CDFADefinition definition) {
                         Safe.ArgumentNotNull(definition, "definition");
                         m_states = definition.States;
                         m_alphabetMap = definition.Alphabet.GetTranslationMap();
                         Feed(new char[0]);
                     }
                     /// <summary>
                     /// Заполняет входными данными буффер.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data) {
                         Safe.ArgumentNotNull(data, "data");
                         Feed(data, data.Length);
                     }
                     /// <summary>
                     /// Заполняет буффур чтения входными данными.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <param name="length">Длина данных для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data, int length) {
                         Safe.ArgumentNotNull(data, "data");
                         Safe.ArgumentInRange(length, 0, data.Length, "length");
                         AssertNotDisposed();
                         m_pointer = -1;
                         m_buffer = data;
                         m_bufferSize = length;
                         Shift();
                     }
                     public void Feed(TextReader reader, bool dispose) {
                         Safe.ArgumentNotNull(reader, "reader");
                         AssertNotDisposed();
                         if (m_reader != null && m_disposeReader)
                             m_reader.Dispose();
                         m_reader = reader;
                         m_disposeReader = dispose;
                         m_pointer = -1;
                         m_buffer = new char[m_chunkSize];
                         m_bufferSize = 0;
                         Shift();
                     }
                     /// <summary>
                     /// Получает текущий токен в виде строки.
                     /// </summary>
                     /// <returns></returns>
                     protected string GetTokenValue() {
                         return new String(m_buffer, m_tokenOffset, m_tokenLen);
                     }
                     /// <summary>
                     /// Метки текущего токена, которые были назначены в регулярном выражении.
                     /// </summary>
                     protected int[] TokenTags {
                         get {
                             return m_currentState.tag;
                         }
                     }
                     /// <summary>
                     /// Признак конца данных
                     /// </summary>
                     public bool EOF {
                         get {
                             return m_pointer >= m_bufferSize;
                         }
                     }
                     /// <summary>
                     /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
                     /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
                     /// котором находится токен.
                     /// </summary>
                     /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
                     protected bool ReadTokenInternal() {
                         if (m_pointer >= m_bufferSize)
                             return false;
                         m_currentState = m_states[CDFADefinition.INITIAL_STATE];
                         m_tokenLen = 0;
                         m_tokenOffset = m_pointer;
                         int nextState = CDFADefinition.UNREACHEBLE_STATE;
                         do {
                             nextState = m_currentState.transitions[m_previewCode];
                             if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
                                 if (m_currentState.final)
                                     return true;
                                 else
                                     throw new ParserException(
                                         String.Format(
                                             "Unexpected symbol '{0}', at pos {1}",
                                             m_buffer[m_pointer],
                                             Position
                                         )
                                     );
                             } else {
                                 m_currentState = m_states[nextState];
                                 m_tokenLen++;
                             }
                         } while (Shift());
                         // END OF DATA
                         if (!m_currentState.final)
                             throw new ParserException("Unexpected end of data");
                         return true;
                     }
                     bool Shift() {
                         m_pointer++;
                         if (m_pointer >= m_bufferSize) {
-                            return ReadNextChunk();
+                            if (!ReadNextChunk())
+                                return false;
                         }
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                         return true;
                     }
                     bool ReadNextChunk() {
                         if (m_reader == null)
                             return false;
                         //  extend buffer if nesessary
                         if (m_pointer + m_chunkSize > m_buffer.Length) {
                             // trim unused buffer head
                             var size = m_tokenLen + m_chunkSize;
                             if (size >= m_limit)
                                 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
                             var temp = new char[size];
                             Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
                             m_pointer -= m_tokenOffset;
                             m_bufferSize -= m_tokenOffset;
                             m_tokenOffset = 0;
                             m_buffer = temp;
                         }
                         var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
                         if (read == 0)
                             return false;
                         m_bufferSize += read;
                         return true;
                     }
                     /// <summary>
                     /// Позиция сканнера во входном буфере
                     /// </summary>
                     public int Position {
                         get {
                             return m_pointer + 1;
                         }
                     }
                     /// <summary>
                     /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
                     /// группировки.
                     /// </summary>
                     /// <param name="states">Таблица состояний нового ДКА</param>
                     /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
                     protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
                         Safe.ArgumentNotNull(states, "dfa");
                         m_defs.Push(new ScannerConfig {
                             states = m_states,
                             alphabetMap = m_alphabetMap
                         });
                         m_states = states;
                         m_alphabetMap = alphabet;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
                     /// <summary>
                     /// Восстанавливает предыдущей ДКА сканнера.
                     /// </summary>
                     protected void Restore() {
                         if (m_defs.Count == 0)
                             throw new InvalidOperationException();
                         var prev = m_defs.Pop();
                         m_states = prev.states;
                         m_alphabetMap = prev.alphabetMap;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing) {
                             if (m_reader != null && m_disposeReader)
                                 m_reader.Dispose();
                             m_buffer = null;
                             m_bufferSize = 0;
                             m_pointer = 0;
                             m_tokenLen = 0;
                             m_tokenOffset = 0;
                         }
                         base.Dispose(disposing);
                     }
                 }
             }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages