implab/ImplabNet Commit - r79:05e6468f066f

sync

cin -

r79:05e6468f066f v2

parent child

Context file:

r79:05e6468f066f

Collapse all files

Implab/JSON/JSONXmlReader.cs +16 0

             using Implab;
             using Implab.Parsing;
             using System;
             using System.Collections.Generic;
             using System.Globalization;
             using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             using System.Xml;
             namespace Implab.JSON {
                 public class JSONXmlReader : XmlReader {
                     enum ValueContext {
                         Undefined,
                         ElementStart,
                         ElementValue,
                         ElementEnd,
                         ElementEmpty
                     }
                     struct LocalNameContext {
                         public string localName;
                         public bool isArray;
                     }
                     JSONParser m_parser;
                     ValueContext m_valueContext;
                     ReadState m_state = ReadState.Initial;
                     Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
                     LocalNameContext m_localName;
                     int m_depthCorrection = 0;
                     readonly string m_rootName;
                     readonly string m_prefix;
                     readonly string m_namespaceUri;
                     readonly bool m_flattenArrays;
                     readonly string m_arrayItemName;
                     readonly XmlNameTable m_nameTable;
                     JSONXmlReader(JSONParser parser, JSONXmlReaderOptions options) {
                         m_parser = parser;
                         if (options != null) {
                             m_prefix = options.NodesPrefix ?? String.Empty;
                             m_namespaceUri = options.NamespaceURI ?? String.Empty;
                             m_rootName = options.RootName ?? "json";
                             m_flattenArrays = options.FlattenArrays;
                             m_arrayItemName = options.ArrayItemName ?? "item";
                             m_nameTable = options.NameTable ?? new NameTable();
                         } else {
                             m_prefix = String.Empty;
                             m_namespaceUri = String.Empty;
                             m_rootName = "json";
                             m_flattenArrays = false;
                             m_arrayItemName = "item";
                             m_nameTable = new NameTable();
                         }
                     }
                     /// <summary>
                     /// Always 0, JSON doesn't support attributes
                     /// </summary>
                     public override int AttributeCount {
                         get { return 0; }
                     }
                     public override string BaseURI {
                         get { return String.Empty; }
                     }
                     public override int Depth {
                         get {
                             return m_localNameStack.Count + m_depthCorrection;
                         }
                     }
                     public override bool EOF {
                         get { return m_parser.EOF; }
                     }
                     /// <summary>
                     /// Always throws an exception
                     /// </summary>
                     /// <param name="i"></param>
                     /// <returns></returns>
                     public override string GetAttribute(int i) {
                         throw new ArgumentOutOfRangeException();
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <param name="namespaceURI"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name, string namespaceURI) {
                         return String.Empty;
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name) {
                         return String.Empty;
                     }
                     public override bool IsEmptyElement {
                         get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
                     }
                     public override string LocalName {
                         get { return m_localName.localName; }
                     }
                     public override string LookupNamespace(string prefix) {
                         if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
                             return m_namespaceUri;
                         else
                             return String.Empty;
                     }
                     public override bool MoveToAttribute(string name, string ns) {
                         return false;
                     }
                     public override bool MoveToAttribute(string name) {
                         return false;
                     }
                     public override bool MoveToElement() {
                         return false;
                     }
                     public override bool MoveToFirstAttribute() {
                         return false;
                     }
                     public override bool MoveToNextAttribute() {
                         return false;
                     }
                     public override XmlNameTable NameTable {
                         get { return m_nameTable; }
                     }
                     public override string NamespaceURI {
                         get { return m_namespaceUri; }
                     }
                     public override XmlNodeType NodeType {
                         get {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.BeginObject:
                                 case JSONElementType.BeginArray:
                                     return XmlNodeType.Element;
                                 case JSONElementType.EndObject:
                                 case JSONElementType.EndArray:
                                     return XmlNodeType.EndElement;
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                         case ValueContext.ElementEmpty:
                                             return XmlNodeType.Element;
                                         case ValueContext.ElementValue:
                                             return XmlNodeType.Text;
                                         case ValueContext.ElementEnd:
                                             return XmlNodeType.EndElement;
                                         default:
                                             throw new InvalidOperationException();
                                     }
                                 default:
                                     throw new InvalidOperationException();
                             }
                         }
                     }
                     public override string Prefix {
                         get { return m_prefix; }
                     }
                     public override bool Read() {
                         if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
                             return false;
                         if (m_state == ReadState.Initial)
                             m_state = System.Xml.ReadState.Interactive;
                         try {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                             SetLocalName(String.Empty);
                                             m_valueContext = ValueContext.ElementValue;
                                             return true;
                                         case ValueContext.ElementValue:
                                             RestoreLocalName();
                                             m_valueContext = ValueContext.ElementEnd;
                                             return true;
                                         case ValueContext.ElementEmpty:
                                         case ValueContext.ElementEnd:
                                             RestoreLocalName();
                                             break;
                                     }
                                     break;
                                 case JSONElementType.EndArray:
                                 case JSONElementType.EndObject:
                                     RestoreLocalName();
                                     break;
                             }
                             string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : m_arrayItemName;
                             while (m_parser.Read()) {
                                 if (!String.IsNullOrEmpty(m_parser.ElementName))
                                     itemName = m_parser.ElementName;
                                 switch (m_parser.ElementType) {
                                     case JSONElementType.BeginArray:
                                         if (m_flattenArrays && !m_localName.isArray) {
                                             m_depthCorrection--;
                                             SetLocalName(itemName, true);
                                             continue;
                                         } else {
                                             SetLocalName(itemName, true);
                                         }
                                         break;
                                     case JSONElementType.BeginObject:
                                         SetLocalName(itemName);
                                         break;
                                     case JSONElementType.EndArray:
                                         if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
                                             RestoreLocalName();
                                             m_depthCorrection++;
                                             continue;
                                         }
                                         break;
                                     case JSONElementType.EndObject:
                                         break;
                                     case JSONElementType.Value:
                                         SetLocalName(itemName);
                                         m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
                                         break;
                                     default:
                                         break;
                                 }
                                 return true;
                             }
                             m_state = System.Xml.ReadState.EndOfFile;
                             return false;
                         } catch {
                             m_state = System.Xml.ReadState.Error;
                             throw;
                         }
                     }
                     public override bool ReadAttributeValue() {
                         return false;
                     }
                     public override ReadState ReadState {
                         get { return m_state; }
                     }
                     public override void ResolveEntity() {
                         // do nothing
                     }
                     public override string Value {
                         get {
                             if (m_parser.ElementValue == null)
                                 return String.Empty;
                             if (Convert.GetTypeCode(m_parser.ElementValue) == TypeCode.Double)
                                 return ((double)m_parser.ElementValue).ToString(CultureInfo.InvariantCulture);
                             else
                                 return m_parser.ElementValue.ToString();
                         }
                     }
                     void SetLocalName(string name) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = false;
                     }
                     void SetLocalName(string name, bool isArray) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = isArray;
                     }
                     void RestoreLocalName() {
                         m_localName = m_localNameStack.Pop();
                     }
                     public override void Close() {
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing) {
                             m_parser.Dispose();
                         }
                         base.Dispose(disposing);
                     }
                     public static JSONXmlReader Create(string file, JSONXmlReaderOptions options) {
                         return Create(File.OpenText(file), options);
                     }
+                    /// <summary>
+                    /// Creates the XmlReader for the specified text stream with JSON data.
+                    /// </summary>
+                    /// <param name="reader">Text reader.</param>
+                    /// <param name="options">Options.</param>
+                    /// <remarks>
+                    /// The reader will be disposed when the XmlReader is disposed.
+                    /// </remarks>
                     public static JSONXmlReader Create(TextReader reader,  JSONXmlReaderOptions options) {
                         return new JSONXmlReader(new JSONParser(reader, true), options);
                     }
+                    /// <summary>
+                    /// Creates the XmlReader for the specified stream with JSON data.
+                    /// </summary>
+                    /// <param name="stream">Stream.</param>
+                    /// <param name="options">Options.</param>
+                    /// <remarks>
+                    /// The stream will be disposed when the XmlReader is disposed.
+                    /// </remarks>
                     public static JSONXmlReader Create(Stream stream,  JSONXmlReaderOptions options) {
                         Safe.ArgumentNotNull(stream, "stream");
                         // HACK don't dispose StreaReader to keep stream opened
                         return Create(new StreamReader(stream), options);
                     }
                 }
             }

Implab/Parsing/Scanner.cs +1 -1

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 /// <summary>
                 /// Базовый класс для разбора потока входных символов на токены.
                 /// </summary>
                 /// <remarks>
                 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
                 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
                 /// конца токена и допустимости текущего символа.
                 /// </remarks>
                 public abstract class Scanner : Disposable {
                     struct ScannerConfig {
                         public DFAStateDescriptior[] states;
                         public int[] alphabetMap;
                     }
                     Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
                     DFAStateDescriptior[] m_states;
                     int[] m_alphabetMap;
                     protected DFAStateDescriptior m_currentState;
                     int m_previewCode;
                     protected int m_tokenLen = 0;
                     protected int m_tokenOffset;
                     protected char[] m_buffer;
                     protected int m_bufferSize;
                     protected int m_pointer;
                     TextReader m_reader;
                     bool m_disposeReader;
                     int m_chunkSize = 1024; // 1k
                     int m_limit = 10 * 1024 * 1024; // 10Mb
-                    public Scanner(CDFADefinition definition) {
+                    protected Scanner(CDFADefinition definition) {
                         Safe.ArgumentNotNull(definition, "definition");
                         m_states = definition.States;
                         m_alphabetMap = definition.Alphabet.GetTranslationMap();
                         Feed(new char[0]);
                     }
                     /// <summary>
                     /// Заполняет входными данными буффер.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data) {
                         Safe.ArgumentNotNull(data, "data");
                         Feed(data, data.Length);
                     }
                     /// <summary>
                     /// Заполняет буффур чтения входными данными.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <param name="length">Длина данных для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data, int length) {
                         Safe.ArgumentNotNull(data, "data");
                         Safe.ArgumentInRange(length, 0, data.Length, "length");
                         AssertNotDisposed();
                         m_pointer = -1;
                         m_buffer = data;
                         m_bufferSize = length;
                         Shift();
                     }
                     public void Feed(TextReader reader, bool dispose) {
                         Safe.ArgumentNotNull(reader, "reader");
                         AssertNotDisposed();
                         if (m_reader != null && m_disposeReader)
                             m_reader.Dispose();
                         m_reader = reader;
                         m_disposeReader = dispose;
                         m_pointer = -1;
                         m_buffer = new char[m_chunkSize];
                         m_bufferSize = 0;
                         Shift();
                     }
                     /// <summary>
                     /// Получает текущий токен в виде строки.
                     /// </summary>
                     /// <returns></returns>
                     protected string GetTokenValue() {
                         return new String(m_buffer, m_tokenOffset, m_tokenLen);
                     }
                     /// <summary>
                     /// Метки текущего токена, которые были назначены в регулярном выражении.
                     /// </summary>
                     protected int[] TokenTags {
                         get {
                             return m_currentState.tag;
                         }
                     }
                     /// <summary>
                     /// Признак конца данных
                     /// </summary>
                     public bool EOF {
                         get {
                             return m_pointer >= m_bufferSize;
                         }
                     }
                     /// <summary>
                     /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
                     /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
                     /// котором находится токен.
                     /// </summary>
                     /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
                     protected bool ReadTokenInternal() {
                         if (m_pointer >= m_bufferSize)
                             return false;
                         m_currentState = m_states[CDFADefinition.INITIAL_STATE];
                         m_tokenLen = 0;
                         m_tokenOffset = m_pointer;
                         int nextState = CDFADefinition.UNREACHEBLE_STATE;
                         do {
                             nextState = m_currentState.transitions[m_previewCode];
                             if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
                                 if (m_currentState.final)
                                     return true;
                                 else
                                     throw new ParserException(
                                         String.Format(
                                             "Unexpected symbol '{0}', at pos {1}",
                                             m_buffer[m_pointer],
                                             Position
                                         )
                                     );
                             } else {
                                 m_currentState = m_states[nextState];
                                 m_tokenLen++;
                             }
                         } while (Shift());
                         // END OF DATA
                         if (!m_currentState.final)
                             throw new ParserException("Unexpected end of data");
                         return true;
                     }
                     bool Shift() {
                         m_pointer++;
                         if (m_pointer >= m_bufferSize) {
                             if (!ReadNextChunk())
                                 return false;
                         }
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                         return true;
                     }
                     bool ReadNextChunk() {
                         if (m_reader == null)
                             return false;
                         //  extend buffer if nesessary
                         if (m_pointer + m_chunkSize > m_buffer.Length) {
                             // trim unused buffer head
                             var size = m_tokenLen + m_chunkSize;
                             if (size >= m_limit)
                                 throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
                             var temp = new char[size];
                             Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
                             m_pointer -= m_tokenOffset;
                             m_bufferSize -= m_tokenOffset;
                             m_tokenOffset = 0;
                             m_buffer = temp;
                         }
                         var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
                         if (read == 0)
                             return false;
                         m_bufferSize += read;
                         return true;
                     }
                     /// <summary>
                     /// Позиция сканнера во входном буфере
                     /// </summary>
                     public int Position {
                         get {
                             return m_pointer + 1;
                         }
                     }
                     /// <summary>
                     /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
                     /// группировки.
                     /// </summary>
                     /// <param name="states">Таблица состояний нового ДКА</param>
                     /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
                     protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
                         Safe.ArgumentNotNull(states, "dfa");
                         m_defs.Push(new ScannerConfig {
                             states = m_states,
                             alphabetMap = m_alphabetMap
                         });
                         m_states = states;
                         m_alphabetMap = alphabet;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
                     /// <summary>
                     /// Восстанавливает предыдущей ДКА сканнера.
                     /// </summary>
                     protected void Restore() {
                         if (m_defs.Count == 0)
                             throw new InvalidOperationException();
                         var prev = m_defs.Pop();
                         m_states = prev.states;
                         m_alphabetMap = prev.alphabetMap;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
                     protected override void Dispose(bool disposing) {
                         if (disposing) {
                             if (m_reader != null && m_disposeReader)
                                 m_reader.Dispose();
                             m_buffer = null;
                             m_bufferSize = 0;
                             m_pointer = 0;
                             m_tokenLen = 0;
                             m_tokenOffset = 0;
                         }
                         base.Dispose(disposing);
                     }
                 }
             }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages