implab/ImplabNet Files · Implab/Formats/JSON/JSONGrammar.cs

DFA refactoring, rx based dfa.

cin - - Load All Authors

File last commit:

r165:e227e78d72e4 ref20160224


                r170:181119ef3b39

ref20160224

Download file

             JSONGrammar.cs
        
                    115 lines
            
             | 4.5 KiB
            
                | text/x-csharp
            
             |
                CSharpLexer
            
             / Implab / Formats / JSON / JSONGrammar.cs
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        cin
    
JSON moved to Formats namespace...

              r163
            
      using System.Linq;

      using Implab.Automaton.RegularExpressions;

        cin
    
DFA refactoring

              r165
            
      using System;

        cin
    
JSON moved to Formats namespace...

              r163
            
      namespace Implab.Formats.JSON {

        cin
    
DFA refactoring

              r165
            
          class JSONGrammar : Grammar<char,JSONGrammar.TokenType> {

        cin
    
JSON moved to Formats namespace...

              r163
            
              public enum TokenType {

                  None,

                  BeginObject,

                  EndObject,

                  BeginArray,

                  EndArray,

                  String,

                  Number,

                  Literal,

                  NameSeparator,

                  ValueSeparator,

                  StringBound,

                  EscapedChar,

                  UnescapedChar,

                  EscapedUnicode,

                  Minus,

                  Plus,

                  Sign,

                  Integer,

                  Dot,

                  Exp

              }

        cin
    
DFA refactoring

              r165
            
              static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();

              public static JSONGrammar Instance {

                  get { return _instance.Value; }

              }

              readonly RegularCharDFADefinition<TokenType> m_jsonDFA;

              readonly RegularCharDFADefinition<TokenType> m_stringDFA;

        cin
    
JSON moved to Formats namespace...

              r163
            
              public JSONGrammar() {

                  DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));

                  var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));

                  var digit9 = SymbolRangeToken('1', '9');

                  var zero = SymbolToken('0');

                  var digit = zero.Or(digit9);

                  var dot = SymbolToken('.');

                  var minus = SymbolToken('-');

                  var sign = SymbolSetToken('-', '+');

                  var expSign = SymbolSetToken('e', 'E');

                  var letters = SymbolRangeToken('a', 'z');

                  var integer = zero.Or(digit9.Cat(digit.EClosure()));

                  var frac = dot.Cat(digit.Closure());

                  var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());

                  var quote = SymbolToken('"');

                  var backSlash = SymbolToken('\\');

                  var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');

                  var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));

                  var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();

                  var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);

                  var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);

                  var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);

                  var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);

                  var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);

                  var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);

                  var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());

                  var literal = letters.Closure();

                  var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));

                  var jsonExpression =

                      number.Tag(TokenType.Number)

                      .Or(literal.Tag(TokenType.Literal))

                      .Or(quote.Tag(TokenType.StringBound))

                      .Or(beginObject.Tag(TokenType.BeginObject))

                      .Or(endObject.Tag(TokenType.EndObject))

                      .Or(beginArray.Tag(TokenType.BeginArray))

                      .Or(endArray.Tag(TokenType.EndArray))

                      .Or(nameSep.Tag(TokenType.NameSeparator))

                      .Or(valueSep.Tag(TokenType.ValueSeparator));

                  var jsonStringExpression =

                      quote.Tag(TokenType.StringBound)

                      .Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))

                      .Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))

                      .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));

        cin
    
DFA refactoring

              r165
            
                  m_jsonDFA = new RegularCharDFADefinition<TokenType>(new CharAlphabet()); 

                  BuildDFA(jsonExpression, m_jsonDFA, m_jsonDFA.InputAlphabet);

                  m_stringDFA = new RegularCharDFADefinition<TokenType>(new CharAlphabet());

                  BuildDFA(jsonStringExpression, m_jsonDFA, m_jsonDFA.InputAlphabet);

        cin
    
JSON moved to Formats namespace...

              r163
            
              }

        cin
    
DFA refactoring

              r165
            
              public RegularCharDFADefinition<TokenType> JsonDFA {

        cin
    
JSON moved to Formats namespace...

              r163
            
                  get {

                      return m_jsonDFA;

                  }

              }

        cin
    
DFA refactoring

              r165
            
              public RegularDFADefinition<char,TokenType> JsonStringDFA {

        cin
    
JSON moved to Formats namespace...

              r163
            
                  get {

                      return m_stringDFA;

                  }

              }

        cin
    
DFA refactoring

              r165
            
              Token<TokenType> SymbolRangeToken(char start, char stop) {

                  return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());

              }

        cin
    
JSON moved to Formats namespace...

              r163
            
          }

      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

cin JSON moved to Formats namespace...	r163	using System.Linq;
		using Implab.Automaton.RegularExpressions;
cin DFA refactoring	r165	using System;
cin JSON moved to Formats namespace...	r163
		namespace Implab.Formats.JSON {
cin DFA refactoring	r165	class JSONGrammar : Grammar<char,JSONGrammar.TokenType> {
cin JSON moved to Formats namespace...	r163	public enum TokenType {
		None,
		BeginObject,
		EndObject,
		BeginArray,
		EndArray,
		String,
		Number,
		Literal,
		NameSeparator,
		ValueSeparator,

		StringBound,
		EscapedChar,
		UnescapedChar,
		EscapedUnicode,

		Minus,
		Plus,
		Sign,
		Integer,
		Dot,
		Exp
		}

cin DFA refactoring	r165	static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();

		public static JSONGrammar Instance {
		get { return _instance.Value; }
		}

		readonly RegularCharDFADefinition<TokenType> m_jsonDFA;
		readonly RegularCharDFADefinition<TokenType> m_stringDFA;
cin JSON moved to Formats namespace...	r163
		public JSONGrammar() {
		DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
		var hexDigit = SymbolRangeToken('a','f').Or(SymbolRangeToken('A','F')).Or(SymbolRangeToken('0','9'));
		var digit9 = SymbolRangeToken('1', '9');
		var zero = SymbolToken('0');
		var digit = zero.Or(digit9);
		var dot = SymbolToken('.');
		var minus = SymbolToken('-');
		var sign = SymbolSetToken('-', '+');
		var expSign = SymbolSetToken('e', 'E');
		var letters = SymbolRangeToken('a', 'z');
		var integer = zero.Or(digit9.Cat(digit.EClosure()));
		var frac = dot.Cat(digit.Closure());
		var exp = expSign.Cat(sign.Optional()).Cat(digit.Closure());
		var quote = SymbolToken('"');
		var backSlash = SymbolToken('\\');
		var specialEscapeChars = SymbolSetToken('\\', '"', '/', 'b', 'f', 't', 'n', 'r');
		var unicodeEspace = SymbolToken('u').Cat(hexDigit.Repeat(4));
		var whitespace = SymbolSetToken('\n', '\r', '\t', ' ').EClosure();
		var beginObject = whitespace.Cat(SymbolToken('{')).Cat(whitespace);
		var endObject = whitespace.Cat(SymbolToken('}')).Cat(whitespace);
		var beginArray = whitespace.Cat(SymbolToken('[')).Cat(whitespace);
		var endArray = whitespace.Cat(SymbolToken(']')).Cat(whitespace);
		var nameSep = whitespace.Cat(SymbolToken(':')).Cat(whitespace);
		var valueSep = whitespace.Cat(SymbolToken(',')).Cat(whitespace);

		var number = minus.Optional().Cat(integer).Cat(frac.Optional()).Cat(exp.Optional());
		var literal = letters.Closure();
		var unescaped = SymbolTokenExcept(Enumerable.Range(0, 0x20).Union(new int[] { '\\', '"' }).Select(x => (char)x));

		var jsonExpression =
		number.Tag(TokenType.Number)
		.Or(literal.Tag(TokenType.Literal))
		.Or(quote.Tag(TokenType.StringBound))
		.Or(beginObject.Tag(TokenType.BeginObject))
		.Or(endObject.Tag(TokenType.EndObject))
		.Or(beginArray.Tag(TokenType.BeginArray))
		.Or(endArray.Tag(TokenType.EndArray))
		.Or(nameSep.Tag(TokenType.NameSeparator))
		.Or(valueSep.Tag(TokenType.ValueSeparator));


		var jsonStringExpression =
		quote.Tag(TokenType.StringBound)
		.Or(backSlash.Cat(specialEscapeChars).Tag(TokenType.EscapedChar))
		.Or(backSlash.Cat(unicodeEspace).Tag(TokenType.EscapedUnicode))
		.Or(unescaped.Closure().Tag(TokenType.UnescapedChar));


cin DFA refactoring	r165	m_jsonDFA = new RegularCharDFADefinition<TokenType>(new CharAlphabet());
		BuildDFA(jsonExpression, m_jsonDFA, m_jsonDFA.InputAlphabet);


		m_stringDFA = new RegularCharDFADefinition<TokenType>(new CharAlphabet());
		BuildDFA(jsonStringExpression, m_jsonDFA, m_jsonDFA.InputAlphabet);
cin JSON moved to Formats namespace...	r163	}

cin DFA refactoring	r165	public RegularCharDFADefinition<TokenType> JsonDFA {
cin JSON moved to Formats namespace...	r163	get {
		return m_jsonDFA;
		}
		}

cin DFA refactoring	r165	public RegularDFADefinition<char,TokenType> JsonStringDFA {
cin JSON moved to Formats namespace...	r163	get {
		return m_stringDFA;
		}
		}
cin DFA refactoring	r165
		Token<TokenType> SymbolRangeToken(char start, char stop) {
		return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
		}

cin JSON moved to Formats namespace...	r163	}
		}