implab/ImplabNet Commit - r59:21611344d366

code cleanup

cin -

r59:21611344d366 default

parent child

Context file:

r59:21611344d366

Collapse all files

Implab/Implab.csproj +1 0

             <?xml version="1.0" encoding="utf-8"?>
             <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
               <PropertyGroup>
                 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
                 <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
                 <ProductVersion>10.0.0</ProductVersion>
                 <SchemaVersion>2.0</SchemaVersion>
                 <ProjectGuid>{F550F1F8-8746-4AD0-9614-855F4C4B7F05}</ProjectGuid>
                 <OutputType>Library</OutputType>
                 <RootNamespace>Implab</RootNamespace>
                 <AssemblyName>Implab</AssemblyName>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
                 <DebugSymbols>true</DebugSymbols>
                 <DebugType>full</DebugType>
                 <Optimize>false</Optimize>
                 <OutputPath>bin\Debug</OutputPath>
                 <DefineConstants>TRACE;DEBUG;</DefineConstants>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
+                <RunCodeAnalysis>true</RunCodeAnalysis>
               </PropertyGroup>
               <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
                 <DebugType>full</DebugType>
                 <Optimize>true</Optimize>
                 <OutputPath>bin\Release</OutputPath>
                 <ErrorReport>prompt</ErrorReport>
                 <WarningLevel>4</WarningLevel>
                 <ConsolePause>false</ConsolePause>
               </PropertyGroup>
               <ItemGroup>
                 <Reference Include="System" />
                 <Reference Include="System.XML" />
               </ItemGroup>
               <ItemGroup>
                 <Compile Include="Component.cs" />
                 <Compile Include="CustomEqualityComparer.cs" />
                 <Compile Include="Diagnostics\ConsoleTraceListener.cs" />
                 <Compile Include="Diagnostics\EventText.cs" />
                 <Compile Include="Diagnostics\IEventTextFormatter.cs" />
                 <Compile Include="Diagnostics\LogChannel.cs" />
                 <Compile Include="Diagnostics\LogicalOperation.cs" />
                 <Compile Include="Diagnostics\TextFileListener.cs" />
                 <Compile Include="Diagnostics\TextListenerBase.cs" />
                 <Compile Include="Diagnostics\TraceLog.cs" />
                 <Compile Include="Diagnostics\TraceContext.cs" />
                 <Compile Include="Diagnostics\TraceEvent.cs" />
                 <Compile Include="Diagnostics\TraceEventType.cs" />
                 <Compile Include="Disposable.cs" />
                 <Compile Include="ICancellable.cs" />
                 <Compile Include="IProgressHandler.cs" />
                 <Compile Include="IProgressNotifier.cs" />
                 <Compile Include="IPromise.cs" />
                 <Compile Include="IPromiseBase.cs" />
                 <Compile Include="IServiceLocator.cs" />
                 <Compile Include="ITaskController.cs" />
                 <Compile Include="JSON\JSONElementContext.cs" />
                 <Compile Include="JSON\JSONElementType.cs" />
                 <Compile Include="JSON\JSONGrammar.cs" />
                 <Compile Include="JSON\JSONParser.cs" />
                 <Compile Include="JSON\JSONScanner.cs" />
                 <Compile Include="JSON\JsonTokenType.cs" />
                 <Compile Include="JSON\JSONWriter.cs" />
                 <Compile Include="JSON\JSONXmlReader.cs" />
                 <Compile Include="JSON\StringTranslator.cs" />
                 <Compile Include="Parallels\DispatchPool.cs" />
                 <Compile Include="Parallels\ArrayTraits.cs" />
                 <Compile Include="Parallels\MTQueue.cs" />
                 <Compile Include="Parallels\WorkerPool.cs" />
                 <Compile Include="Parsing\Alphabet.cs" />
                 <Compile Include="Parsing\AlphabetBase.cs" />
                 <Compile Include="Parsing\AltToken.cs" />
                 <Compile Include="Parsing\BinaryToken.cs" />
                 <Compile Include="Parsing\CatToken.cs" />
                 <Compile Include="Parsing\CDFADefinition.cs" />
                 <Compile Include="Parsing\DFABuilder.cs" />
                 <Compile Include="Parsing\DFADefinitionBase.cs" />
                 <Compile Include="Parsing\DFAStateDescriptor.cs" />
                 <Compile Include="Parsing\DFAutomaton.cs" />
                 <Compile Include="Parsing\EDFADefinition.cs" />
                 <Compile Include="Parsing\EmptyToken.cs" />
                 <Compile Include="Parsing\EndToken.cs" />
                 <Compile Include="Parsing\EnumAlphabet.cs" />
                 <Compile Include="Parsing\Grammar.cs" />
                 <Compile Include="Parsing\IAlphabet.cs" />
                 <Compile Include="Parsing\IDFADefinition.cs" />
                 <Compile Include="Parsing\IVisitor.cs" />
                 <Compile Include="Parsing\ParserException.cs" />
                 <Compile Include="Parsing\Scanner.cs" />
                 <Compile Include="Parsing\StarToken.cs" />
                 <Compile Include="Parsing\SymbolToken.cs" />
                 <Compile Include="Parsing\Token.cs" />
                 <Compile Include="ServiceLocator.cs" />
                 <Compile Include="TaskController.cs" />
                 <Compile Include="ProgressInitEventArgs.cs" />
                 <Compile Include="Properties\AssemblyInfo.cs" />
                 <Compile Include="Promise.cs" />
                 <Compile Include="Parallels\AsyncPool.cs" />
                 <Compile Include="Safe.cs" />
                 <Compile Include="ValueEventArgs.cs" />
               </ItemGroup>
               <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
               <ItemGroup />
             </Project>

Implab/JSON/JSONParser.cs +53 -1

             using Implab;
             using Implab.Parsing;
             using System;
             using System.Collections.Generic;
             using System.Diagnostics;
+            using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.JSON {
                 /// <summary>
                 /// internal
                 /// </summary>
                 public struct JSONParserContext {
                     public string memberName;
                     public JSONElementContext elementContext;
                 }
                 /// <summary>
                 /// Pull парсер JSON данных.
                 /// </summary>
-                public class JSONParser : DFAutomaton<JSONParserContext> {
+                public class JSONParser : DFAutomaton<JSONParserContext>, IDisposable {
                     enum MemberContext {
                         MemberName,
                         MemberValue
                     }
                     static readonly EnumAlphabet<JsonTokenType> _alphabet = EnumAlphabet<JsonTokenType>.FullAlphabet;
                     static readonly DFAStateDescriptior[] _jsonDFA;
                     static readonly DFAStateDescriptior[] _objectDFA;
                     static readonly DFAStateDescriptior[] _arrayDFA;
                     static JSONParser() {
                         var jsonExpression = Token.New(JsonTokenType.BeginObject, JsonTokenType.BeginArray).Tag(0);
                         var valueExpression = Token.New(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
                         var memberExpression = Token.New(JsonTokenType.String).Cat(Token.New(JsonTokenType.NameSeparator)).Cat(valueExpression);
                         var objectExpression = memberExpression
                             .Cat(
                                 Token.New(JsonTokenType.ValueSeparator)
                                 .Cat(memberExpression)
                                 .EClosure()
                             )
                             .Optional()
                             .Cat(Token.New(JsonTokenType.EndObject))
                             .Tag(0);
                         var arrayExpression = valueExpression
                             .Cat(
                                 Token.New(JsonTokenType.ValueSeparator)
                                 .Cat(valueExpression)
                                 .EClosure()
                             )
                             .Optional()
                             .Cat(Token.New(JsonTokenType.EndArray))
                             .Tag(0);
                         _jsonDFA = BuildDFA(jsonExpression).States;
                         _objectDFA = BuildDFA(objectExpression).States;
                         _arrayDFA = BuildDFA(arrayExpression).States;
                     }
                     static EDFADefinition<JsonTokenType> BuildDFA(Token expr) {
                         var builder = new DFABuilder();
                         var dfa = new EDFADefinition<JsonTokenType>(_alphabet);
                         expr.Accept(builder);
                         builder.BuildDFA(dfa);
                         return dfa;
                     }
                     JSONScanner m_scanner;
                     MemberContext m_memberContext;
                     JSONElementType m_elementType;
                     object m_elementValue;
+                    /// <summary>
+                    /// Создает новый парсер на основе строки, содержащей JSON
+                    /// </summary>
+                    /// <param name="text"></param>
                     public JSONParser(string text)
                         : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
                         Safe.ArgumentNotEmpty(text, "text");
                         m_scanner = new JSONScanner();
                         m_scanner.Feed(text.ToCharArray());
                     }
+                    /// <summary>
+                    /// Создает новый экземпляр парсера, на основе текстового потока.
+                    /// </summary>
+                    /// <param name="reader">Текстовый поток.</param>
+                    /// <param name="dispose">Признак того, что парсер должен конролировать время жизни входного потока.</param>
+                    public JSONParser(TextReader reader, bool dispose)
+                        : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
+                        Safe.ArgumentNotNull(reader, "reader");
+                        m_scanner = new JSONScanner();
+                        m_scanner.Feed(reader, dispose);
+                    }
+                    /// <summary>
+                    /// Тип текущего элемента на котором стоит парсер.
+                    /// </summary>
                     public JSONElementType ElementType {
                         get { return m_elementType; }
                     }
+                    /// <summary>
+                    /// Имя элемента - имя свойства родительского контейнера. Для элементов массивов и корневого всегда
+                    /// пустая строка.
+                    /// </summary>
                     public string ElementName {
                         get { return m_context.info.memberName; }
                     }
+                    /// <summary>
+                    /// Значение элемента. Только для элементов типа <see cref="JSONElementType.Value"/>, для остальных <c>null</c>
+                    /// </summary>
                     public object ElementValue {
                         get { return m_elementValue; }
                     }
+                    /// <summary>
+                    /// Читает слеюудущий объект из потока
+                    /// </summary>
+                    /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
                     public bool Read() {
                         if (m_context.current == UNREACHEBLE_STATE)
                             throw new InvalidOperationException("The parser is in invalid state");
                         object tokenValue;
                         JsonTokenType tokenType;
                         m_context.info.memberName = String.Empty;
                         while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
                             Move((int)tokenType);
                             if (m_context.current == UNREACHEBLE_STATE)
                                 UnexpectedToken(tokenValue, tokenType);
                             switch (tokenType) {
                                 case JsonTokenType.BeginObject:
                                     Switch(
                                         _objectDFA,
                                         INITIAL_STATE,
                                         new JSONParserContext {
                                             memberName = m_context.info.memberName,
                                             elementContext = JSONElementContext.Object
                                         }
                                     );
                                     m_elementValue = null;
                                     m_memberContext = MemberContext.MemberName;
                                     m_elementType = JSONElementType.BeginObject;
                                     return true;
                                 case JsonTokenType.EndObject:
                                     Restore();
                                     m_elementValue = null;
                                     m_elementType = JSONElementType.EndObject;
                                     return true;
                                 case JsonTokenType.BeginArray:
                                     Switch(
                                         _arrayDFA,
                                         INITIAL_STATE,
                                         new JSONParserContext {
                                             memberName = m_context.info.memberName,
                                             elementContext = JSONElementContext.Array
                                         }
                                     );
                                     m_elementValue = null;
                                     m_memberContext = MemberContext.MemberValue;
                                     m_elementType = JSONElementType.BeginArray;
                                     return true;
                                 case JsonTokenType.EndArray:
                                     Restore();
                                     m_elementValue = null;
                                     m_elementType = JSONElementType.EndArray;
                                     return true;
                                 case JsonTokenType.String:
                                     if (m_memberContext == MemberContext.MemberName) {
                                         m_context.info.memberName = (string)tokenValue;
                                         break;
                                     } else {
                                         m_elementType = JSONElementType.Value;
                                         m_elementValue = tokenValue;
                                         return true;
                                     }
                                 case JsonTokenType.Number:
                                     m_elementType = JSONElementType.Value;
                                     m_elementValue = tokenValue;
                                     return true;
                                 case JsonTokenType.Literal:
                                     m_elementType = JSONElementType.Value;
                                     m_elementValue = ParseLiteral((string)tokenValue);
                                     return true;
                                 case JsonTokenType.NameSeparator:
                                     m_memberContext = MemberContext.MemberValue;
                                     break;
                                 case JsonTokenType.ValueSeparator:
                                     m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
                                     break;
                                 default:
                                     UnexpectedToken(tokenValue, tokenType);
                                     break;
                             }
                         }
                         if (m_context.info.elementContext != JSONElementContext.None)
                             throw new ParserException("Unexpedted end of data");
                         return false;
                     }
                     object ParseLiteral(string literal) {
                         switch (literal) {
                             case "null":
                                 return null;
                             case "false":
                                 return false;
                             case "true":
                                 return true;
                             default:
                                 UnexpectedToken(literal, JsonTokenType.Literal);
                                 return null; // avoid compliler error
                         }
                     }
                     void UnexpectedToken(object value, JsonTokenType tokenType) {
                         throw new ParserException(String.Format("Unexpected token {0}: '{1}'", tokenType, value));
                     }
+                    /// <summary>
+                    /// Признак конца потока
+                    /// </summary>
                     public bool EOF {
                         get {
                             return m_scanner.EOF;
                         }
                     }
+                    protected virtual void Dispose(bool disposing) {
+                        if (disposing) {
+                            m_scanner.Dispose();
+                        }
+                    }
+                    /// <summary>
+                    /// Освобождает парсер и связанный с ним сканнер.
+                    /// </summary>
+                    public void Dispose() {
+                        Dispose(true);
+                        GC.SuppressFinalize(this);
+                    }
+                    ~JSONParser() {
+                        Dispose(false);
+                    }
                 }
             }

Implab/JSON/JSONXmlReader.cs +10 -2

             using Implab;
             using Implab.JSON;
             using Implab.Parsing;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             using System.Xml;
             namespace ConsPlay {
                 public class JSONXmlReader : XmlReader {
                     enum ValueContext {
                         Undefined,
                         ElementStart,
                         ElementValue,
                         ElementEnd,
                         ElementEmpty
                     }
                     struct LocalNameContext {
                         public string localName;
                         public bool isArray;
                     }
                     JSONParser m_parser;
                     ValueContext m_valueContext;
                     ReadState m_state = ReadState.Initial;
                     Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
                     LocalNameContext m_localName;
                     string m_rootName = "json";
                     string m_prefix = String.Empty;
                     string m_namespaceUri = String.Empty;
                     bool m_flattenArrays = false;
                     NameTable m_nameTable = new NameTable();
                     int m_depthCorrection = 0;
                     public JSONXmlReader(JSONParser parser) {
                         Safe.ArgumentNotNull(parser, "parser");
                         m_parser = parser;
                     }
                     /// <summary>
                     /// Always 0, JSON doesn't support attributes
                     /// </summary>
                     public override int AttributeCount {
                         get { return 0; }
                     }
                     public override string BaseURI {
                         get { return String.Empty; }
                     }
                     public override int Depth {
                         get {
                             return m_localNameStack.Count+m_depthCorrection;
                         }
                     }
                     public override bool EOF {
                         get { return m_parser.EOF; }
                     }
                     /// <summary>
                     /// Always throws an exception
                     /// </summary>
                     /// <param name="i"></param>
                     /// <returns></returns>
                     public override string GetAttribute(int i) {
                         throw new ArgumentOutOfRangeException();
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <param name="namespaceURI"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name, string namespaceURI) {
                         return String.Empty;
                     }
                     /// <summary>
                     /// Always returns empty string
                     /// </summary>
                     /// <param name="name"></param>
                     /// <returns></returns>
                     public override string GetAttribute(string name) {
                         return String.Empty;
                     }
                     public override bool IsEmptyElement {
                         get { return m_parser.ElementType == JSONElementType.Value && m_valueContext == ValueContext.ElementEmpty; }
                     }
                     public override string LocalName {
                         get { return m_localName.localName; }
                     }
                     public override string LookupNamespace(string prefix) {
                         if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
                             return m_namespaceUri;
                         else
                             return String.Empty;
                     }
                     public override bool MoveToAttribute(string name, string ns) {
                         return false;
                     }
                     public override bool MoveToAttribute(string name) {
                         return false;
                     }
                     public override bool MoveToElement() {
                         return false;
                     }
                     public override bool MoveToFirstAttribute() {
                         return false;
                     }
                     public override bool MoveToNextAttribute() {
                         return false;
                     }
                     public override XmlNameTable NameTable {
                         get { return m_nameTable; }
                     }
                     public override string NamespaceURI {
                         get { return m_namespaceUri; }
                     }
                     public override XmlNodeType NodeType {
                         get {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.BeginObject:
                                 case JSONElementType.BeginArray:
                                     return XmlNodeType.Element;
                                 case JSONElementType.EndObject:
                                 case JSONElementType.EndArray:
                                     return XmlNodeType.EndElement;
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                         case ValueContext.ElementEmpty:
                                             return XmlNodeType.Element;
                                         case ValueContext.ElementValue:
                                             return XmlNodeType.Text;
                                         case ValueContext.ElementEnd:
                                             return XmlNodeType.EndElement;
                                         default:
                                             throw new InvalidOperationException();
                                     }
                                 default:
                                     throw new InvalidOperationException();
                             }
                         }
                     }
                     public override string Prefix {
                         get { return m_prefix; }
                     }
                     public override bool Read() {
                         if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
                             return false;
                         if (m_state == ReadState.Initial)
                             m_state = System.Xml.ReadState.Interactive;
                         try {
                             switch (m_parser.ElementType) {
                                 case JSONElementType.Value:
                                     switch (m_valueContext) {
                                         case ValueContext.ElementStart:
                                             SetLocalName(String.Empty);
                                             m_valueContext = ValueContext.ElementValue;
                                             return true;
                                         case ValueContext.ElementValue:
                                             RestoreLocalName();
                                             m_valueContext = ValueContext.ElementEnd;
                                             return true;
                                         case ValueContext.ElementEmpty:
                                         case ValueContext.ElementEnd:
                                             RestoreLocalName();
                                             break;
                                     }
                                     break;
                                 case JSONElementType.EndArray:
                                 case JSONElementType.EndObject:
                                     RestoreLocalName();
                                     break;
                             }
                             string itemName = m_parser.ElementType == JSONElementType.None ? m_rootName : m_flattenArrays ? m_localName.localName : "item";
                             while (m_parser.Read()) {
                                 if (!String.IsNullOrEmpty(m_parser.ElementName))
                                     itemName = m_parser.ElementName;
                                 switch (m_parser.ElementType) {
                                     case JSONElementType.BeginArray:
                                         if (m_flattenArrays && !m_localName.isArray) {
                                             m_depthCorrection--;
                                             SetLocalName(itemName, true);
                                             continue;
                                         } else {
                                             SetLocalName(itemName, true);
                                         }
                                         break;
                                     case JSONElementType.BeginObject:
                                         SetLocalName(itemName);
                                         break;
                                     case JSONElementType.EndArray:
                                         if (m_flattenArrays && !m_localNameStack.Peek().isArray) {
                                             RestoreLocalName();
                                             m_depthCorrection++;
                                             continue;
                                         }
                                         break;
                                     case JSONElementType.EndObject:
                                         break;
                                     case JSONElementType.Value:
                                         SetLocalName(itemName);
                                         m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
                                         break;
                                     default:
                                         break;
                                 }
                                 return true;
                             }
                             m_state = System.Xml.ReadState.EndOfFile;
                             return false;
                         } catch {
                             m_state = System.Xml.ReadState.Error;
                             throw;
                         }
                     }
                     public override bool ReadAttributeValue() {
                         return false;
                     }
                     public override ReadState ReadState {
                         get { return m_state; }
                     }
                     public override void ResolveEntity() {
                         // do nothing
                     }
                     public override string Value {
                         get { return m_parser.ElementValue == null ? String.Empty : m_parser.ElementValue.ToString(); }
                     }
                     void SetLocalName(string name) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = false;
                     }
                     void SetLocalName(string name, bool isArray) {
                         m_localNameStack.Push(m_localName);
                         m_localName.localName = name;
                         m_localName.isArray = isArray;
                     }
                     void RestoreLocalName() {
                         m_localName = m_localNameStack.Pop();
                     }
                     public override void Close() {
-                        m_state = System.Xml.ReadState.EndOfFile ;
+                    }
+                    protected override void Dispose(bool disposing) {
+                        if (disposing) {
+                            m_parser.Dispose();
+                        }
+                        base.Dispose(disposing);
+                    }
                 }
             }

Implab/Parsing/Alphabet.cs +4 -4

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 public class Alphabet: AlphabetBase<char> {
+                    public Alphabet()
+                        : base(char.MaxValue + 1) {
+                    }
                     public override int GetSymbolIndex(char symbol) {
                         return symbol;
                     }
                     public override IEnumerable<char> InputSymbols {
                         get { return Enumerable.Range(char.MinValue, char.MaxValue).Select(x => (char)x); }
                     }
-                    protected override int MapSize {
-                        get { return char.MaxValue + 1; }
                 }
             }

Implab/Parsing/AlphabetBase.cs +2 -5

             using Implab;
             using System;
             using System.Collections.Generic;
             using System.Diagnostics;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 public abstract class AlphabetBase<T> : IAlphabet<T> {
                     public const int UNCLASSIFIED = 0;
                     int m_nextId = 1;
                     int[] m_map;
                     public int Count {
                         get { return m_nextId; }
                     }
-                    protected AlphabetBase() {
+                    protected AlphabetBase(int mapSize) {
-                        m_map = new int[MapSize];
+                        m_map = new int[mapSize];
                     }
                     protected AlphabetBase(int[] map) {
                         Debug.Assert(map != null);
-                        Debug.Assert(map.Length == MapSize);
                         m_map = map;
                         m_nextId = map.Max() + 1;
                     }
                     public int DefineSymbol(T symbol) {
                         var index = GetSymbolIndex(symbol);
                         if (m_map[index] == UNCLASSIFIED)
                             m_map[index] = m_nextId++;
                         return m_map[index];
                     }
                     public int DefineClass(IEnumerable<T> symbols) {
                         Safe.ArgumentNotNull(symbols, "symbols");
                         symbols = symbols.Distinct();
                         foreach (var symbol in symbols) {
                             var index = GetSymbolIndex(symbol);
                             if (m_map[index] == UNCLASSIFIED)
                                 m_map[GetSymbolIndex(symbol)] = m_nextId;
                             else
                                 throw new InvalidOperationException(String.Format("Symbol '{0}' already in use", symbol));
                         }
                         return m_nextId++;
                     }
                     public List<T>[] CreateReverseMap() {
                         return
                             Enumerable.Range(UNCLASSIFIED, Count)
                                 .Select(
                                     i => InputSymbols
                                         .Where(x => i != UNCLASSIFIED && m_map[GetSymbolIndex(x)] == i)
                                         .ToList()
                                 )
                                 .ToArray();
                     }
                     public int[] Reclassify(IAlphabet<T> newAlphabet, IEnumerable<ICollection<int>> classes) {
                         Safe.ArgumentNotNull(newAlphabet, "newAlphabet");
                         Safe.ArgumentNotNull(classes, "classes");
                         var reverseMap = CreateReverseMap();
                         int[] translationMap = new int[Count];
                         foreach (var scl in classes) {
                             // skip if the supper class contains the unclassified element
                             if (scl.Contains(UNCLASSIFIED))
                                 continue;
                             var range = new List<T>();
                             foreach (var cl in scl) {
                                 if (cl < 0 || cl >= reverseMap.Length)
                                     throw new ArgumentOutOfRangeException(String.Format("Class {0} is not valid for the current alphabet", cl));
                                 range.AddRange(reverseMap[cl]);
                             }
                             var newClass = newAlphabet.DefineClass(range);
                             foreach (var cl in scl)
                                 translationMap[cl] = newClass;
                         }
                         return translationMap;
                     }
                     public int Translate(T symbol) {
                         return m_map[GetSymbolIndex(symbol)];
                     }
                     public abstract int GetSymbolIndex(T symbol);
                     public abstract IEnumerable<T> InputSymbols { get; }
-                    protected abstract int MapSize { get; }
                     public int[] GetTranslationMap() {
                         return m_map;
                     }
                 }
             }

Implab/Parsing/EnumAlphabet.cs +3 -4

             using Implab;
             using System;
             using System.Collections.Generic;
+            using System.Diagnostics;
             using System.Globalization;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 /// <summary>
                 /// Алфавит символами которого являются элементы перечислений.
                 /// </summary>
                 /// <typeparam name="T">Тип перечислений</typeparam>
                 public class EnumAlphabet<T> : AlphabetBase<T> where T : struct, IConvertible {
                     static readonly T[] _symbols;
                     static readonly EnumAlphabet<T> _fullAlphabet;
                     static EnumAlphabet() {
                         if (!typeof(T).IsEnum)
                             throw new InvalidOperationException("Invalid generic parameter, enumeration is required");
                         if (Enum.GetUnderlyingType(typeof(T)) != typeof(Int32))
                             throw new InvalidOperationException("Only enums based on Int32 are supported");
                         _symbols = ((T[])Enum.GetValues(typeof(T)))
                             .OrderBy(x => x.ToInt32(CultureInfo.InvariantCulture))
                             .ToArray();
                         if (
                             _symbols[_symbols.Length - 1].ToInt32(CultureInfo.InvariantCulture) >= _symbols.Length
                             || _symbols[0].ToInt32(CultureInfo.InvariantCulture) != 0
                         )
                             throw new InvalidOperationException("The specified enumeration must be zero-based and continuously numbered");
                         _fullAlphabet = new EnumAlphabet<T>(_symbols.Select(x => x.ToInt32(CultureInfo.InvariantCulture)).ToArray());
                     }
                     public static EnumAlphabet<T> FullAlphabet {
                         get {
                             return _fullAlphabet;
                         }
                     }
                     public EnumAlphabet()
-                        : base() {
+                        : base(_symbols.Length) {
                     }
                     public EnumAlphabet(int[] map)
                         : base(map) {
+                        Debug.Assert(map.Length == _symbols.Length);
                     }
                     public override int GetSymbolIndex(T symbol) {
                         return symbol.ToInt32(CultureInfo.InvariantCulture);
                     }
                     public override IEnumerable<T> InputSymbols {
                         get { return _symbols; }
                     }
-                    protected override int MapSize {
-                        get { return _symbols.Length; }
                 }
             }

Implab/Parsing/Scanner.cs +61 -18

             using Implab;
             using System;
             using System.Collections.Generic;
+            using System.IO;
             using System.Linq;
             using System.Text;
             using System.Threading.Tasks;
             namespace Implab.Parsing {
                 /// <summary>
                 /// Базовый класс для разбора потока входных символов на токены.
                 /// </summary>
                 /// <remarks>
                 /// Сканнер имеет внутри буффер с симолами входного текста, по которому перемещаются два
                 /// указателя, начала и конца токена, при перемещении искользуется ДКА для определения
                 /// конца токена и допустимости текущего символа.
                 /// </remarks>
-                public class Scanner {
+                public abstract class Scanner : Disposable {
                     struct ScannerConfig {
                         public DFAStateDescriptior[] states;
                         public int[] alphabetMap;
                     }
                     Stack<ScannerConfig> m_defs = new Stack<ScannerConfig>();
                     DFAStateDescriptior[] m_states;
                     int[] m_alphabetMap;
                     protected DFAStateDescriptior m_currentState;
                     int m_previewCode;
                     protected int m_tokenLen = 0;
                     protected int m_tokenOffset;
                     protected char[] m_buffer;
                     protected int m_bufferSize;
                     protected int m_pointer;
-                    public Scanner(CDFADefinition definition, string text) {
+                    TextReader m_reader;
-                        Safe.ArgumentNotNull(definition, "definition");
+                    bool m_disposeReader;
-                        Safe.ArgumentNotEmpty(text, "text");
+                    int m_chunkSize = 1024; // 1k
+                    int m_limit = 10 * 1024 * 1024; // 10Mb
-                        m_states = definition.States;
-                        m_alphabetMap = definition.Alphabet.GetTranslationMap();
-                        Feed(text.ToCharArray());
                     public Scanner(CDFADefinition definition) {
                         Safe.ArgumentNotNull(definition, "definition");
                         m_states = definition.States;
                         m_alphabetMap = definition.Alphabet.GetTranslationMap();
                         Feed(new char[0]);
                     }
                     /// <summary>
                     /// Заполняет входными данными буффер.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data) {
                         Safe.ArgumentNotNull(data, "data");
                         Feed(data, data.Length);
                     }
                     /// <summary>
                     /// Заполняет буффур чтения входными данными.
                     /// </summary>
                     /// <param name="data">Данные для обработки.</param>
                     /// <param name="length">Длина данных для обработки.</param>
                     /// <remarks>Копирование данных не происходит, переданный массив используется в
                     /// качестве входного буффера.</remarks>
                     public void Feed(char[] data, int length) {
                         Safe.ArgumentNotNull(data, "data");
                         Safe.ArgumentInRange(length, 0, data.Length, "length");
+                        AssertNotDisposed();
                         m_pointer = -1;
                         m_buffer = data;
                         m_bufferSize = length;
                         Shift();
                     }
+                    public void Feed(TextReader reader, bool dispose) {
+                        Safe.ArgumentNotNull(reader, "reader");
+                        AssertNotDisposed();
+                        if (m_reader != null && m_disposeReader)
+                            m_reader.Dispose();
+                        m_reader = reader;
+                        m_disposeReader = dispose;
+                        m_pointer = -1;
+                        m_buffer = new char[m_chunkSize];
+                        m_bufferSize = 0;
+                        Shift();
+                    }
                     /// <summary>
                     /// Получает текущий токен в виде строки.
                     /// </summary>
                     /// <returns></returns>
-                    public string GetTokenValue() {
+                    protected string GetTokenValue() {
                         return new String(m_buffer, m_tokenOffset, m_tokenLen);
                     }
                     /// <summary>
                     /// Метки текущего токена, которые были назначены в регулярном выражении.
                     /// </summary>
-                    public int[] TokenTags {
+                    protected int[] TokenTags {
                         get {
                             return m_currentState.tag;
                         }
                     }
                     /// <summary>
                     /// Признак конца данных
                     /// </summary>
                     public bool EOF {
                         get {
                             return m_pointer >= m_bufferSize;
                         }
                     }
                     /// <summary>
                     /// Читает следующий токен, при этом <see cref="m_tokenOffset"/> указывает на начало токена,
                     /// <see cref="m_tokenLen"/> на длину токена, <see cref="m_buffer"/> - массив символов, в
                     /// котором находится токен.
                     /// </summary>
                     /// <returns><c>false</c> - достигнут конец данных, токен не прочитан.</returns>
                     protected bool ReadTokenInternal() {
                         if (m_pointer >= m_bufferSize)
                             return false;
                         m_currentState = m_states[CDFADefinition.INITIAL_STATE];
                         m_tokenLen = 0;
                         m_tokenOffset = m_pointer;
                         int nextState = CDFADefinition.UNREACHEBLE_STATE;
                         do {
                             nextState = m_currentState.transitions[m_previewCode];
                             if (nextState == CDFADefinition.UNREACHEBLE_STATE) {
                                 if (m_currentState.final)
                                     return true;
                                 else
                                     throw new ParserException(
                                         String.Format(
                                             "Unexpected symbol '{0}', at pos {1}",
                                             m_buffer[m_pointer],
                                             Position
                                         )
                                     );
                             } else {
                                 m_currentState = m_states[nextState];
                                 m_tokenLen++;
                             }
                         } while (Shift());
                         // END OF DATA
                         if (!m_currentState.final)
                             throw new ParserException("Unexpected end of data");
                         return true;
                     }
                     bool Shift() {
                         m_pointer++;
                         if (m_pointer >= m_bufferSize) {
                             return ReadNextChunk();
                         }
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                         return true;
                     }
-                    /// <summary>
+                    bool ReadNextChunk() {
-                    /// Вызывается по достижению конца входного буффера для получения
+                        if (m_reader == null)
-                    /// новых данных.
-                    /// </summary>
-                    /// <returns><c>true</c> - новые двнные получены, можно продолжать обработку.</returns>
-                    protected virtual bool ReadNextChunk() {
                             return false;
+                        //  extend buffer if nesessary
+                        if (m_pointer + m_chunkSize > m_buffer.Length) {
+                            // trim unused buffer head
+                            var size = m_tokenLen + m_chunkSize;
+                            if (size >= m_limit)
+                                throw new ParserException(String.Format("Input buffer {0} bytes limit exceeded", m_limit));
+                            var temp = new char[size];
+                            Array.Copy(m_buffer, m_tokenOffset, temp, 0, m_tokenLen);
+                            m_pointer -= m_tokenOffset;
+                            m_bufferSize -= m_tokenOffset;
+                            m_tokenOffset = 0;
+                            m_buffer = temp;
+                        }
+                        var read = m_reader.Read(m_buffer, m_tokenLen, m_chunkSize);
+                        if (read == 0)
+                            return false;
+                        m_bufferSize += read;
+                        return true;
                     }
                     /// <summary>
                     /// Позиция сканнера во входном буфере
                     /// </summary>
                     public int Position {
                         get {
                             return m_pointer + 1;
                         }
                     }
                     /// <summary>
                     /// Преключает внутренний ДКА на указанный, позволяет реализовать подобие захватывающей
                     /// группировки.
                     /// </summary>
                     /// <param name="states">Таблица состояний нового ДКА</param>
                     /// <param name="alphabet">Таблица входных символов для нового ДКА</param>
                     protected void Switch(DFAStateDescriptior[] states, int[] alphabet) {
                         Safe.ArgumentNotNull(states, "dfa");
                         m_defs.Push(new ScannerConfig {
                             states = m_states,
                             alphabetMap = m_alphabetMap
                         });
                         m_states = states;
                         m_alphabetMap = alphabet;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
                     /// <summary>
                     /// Восстанавливает предыдущей ДКА сканнера.
                     /// </summary>
                     protected void Restore() {
                         if (m_defs.Count == 0)
                             throw new InvalidOperationException();
                         var prev = m_defs.Pop();
                         m_states = prev.states;
                         m_alphabetMap = prev.alphabetMap;
                         m_previewCode = m_alphabetMap[m_buffer[m_pointer]];
                     }
+                    protected override void Dispose(bool disposing) {
+                        if (disposing) {
+                            if (m_reader != null && m_disposeReader)
+                                m_reader.Dispose();
+                            m_buffer = null;
+                            m_bufferSize = 0;
+                            m_pointer = 0;
+                            m_tokenLen = 0;
+                            m_tokenOffset = 0;
+                        }
+                        base.Dispose(disposing);
                     }
                 }
+            }

Implab/Properties/AssemblyInfo.cs +2 0

             using System.Reflection;
             using System.Runtime.CompilerServices;
+            using System.Runtime.InteropServices;
             // Information about this assembly is defined by the following attributes.
             // Change them to the values specific to your project.
             [assembly: AssemblyTitle("Implab")]
             [assembly: AssemblyDescription("Tools")]
             [assembly: AssemblyConfiguration("")]
             [assembly: AssemblyCompany("")]
             [assembly: AssemblyProduct("")]
             [assembly: AssemblyCopyright("Implab")]
             [assembly: AssemblyTrademark("")]
             // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
             // The form "{Major}.{Minor}.*" will automatically update the build and revision,
             // and "{Major}.{Minor}.{Build}.*" will update just the revision.
             [assembly: AssemblyVersion("1.0.*")]
+            [assembly: ComVisible(false)]
             // The following attributes are used to specify the signing key for the assembly,
             // if desired. See the Mono documentation for more information about signing.
             //[assembly: AssemblyDelaySign(false)]
             //[assembly: AssemblyKeyFile("")]

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages