implab/ImplabNet Commit - r236:302ca905c19e

JsonReader optimizations

cin -

r236:302ca905c19e v2

parent child

Context file:

r236:302ca905c19e

Collapse all files

Implab.Format.Test/JsonTests.cs +30 -4

@@ -9,8 +9,8 using System.IO;
9		9
10	namespace Implab.Format.Test {	10	namespace Implab.Format.Test {
11	[TestFixture]	11	[TestFixture]
12	public class JsonTests {	12	public class JsonTests {
13		13
14	[Test]	14	[Test]
15	public void TestScannerValidTokens() {	15	public void TestScannerValidTokens() {
16	using (var scanner = JsonStringScanner.Create(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {	16	using (var scanner = JsonStringScanner.Create(@"9123, -123, 0, 0.1, -0.2, -0.1e3, 1.3E-3, ""some \t\n\u0020 text"", literal []{}:")) {
@@ -114,10 +114,36 namespace Implab.Format.Test {
114	DumpJsonParse("[{\"info\": [7,8,9]}]");	114	DumpJsonParse("[{\"info\": [7,8,9]}]");
115	DumpJsonFlatParse("[1,2,\"\",[3,4],{\"info\": [5,6]},{\"num\": [7,8,null]}, null,[null]]");	115	DumpJsonFlatParse("[1,2,\"\",[3,4],{\"info\": [5,6]},{\"num\": [7,8,null]}, null,[null]]");
116	}	116	}
117		117
		118	[Test]
		119	public void JsonBenchmark() {
		120	var t = Environment.TickCount;
		121	using (var reader = new JsonXmlReader(JsonReader.Create("e:\\citylots.json"), new JsonXmlReaderOptions { NamespaceUri = "XmlReaderSimpleTest", RootName = "data" })) {
		122	while (reader.Read()) {
		123	}
		124	}
		125	Console.WriteLine($"JsonXmlReader: {Environment.TickCount - t} ms");
		126
		127	t = Environment.TickCount;
		128	using(var reader = JsonReader.Create("e:\\citylots.json")) {
		129	while(reader.Read()) {
		130	}
		131	}
		132
		133	Console.WriteLine($"JsonReader: {Environment.TickCount - t} ms");
		134
		135	t = Environment.TickCount;
		136	using (var reader = XmlReader.Create("file:///e:\\citylots.xml")) {
		137	while (reader.Read()) {
		138	}
		139	}
		140
		141	Console.WriteLine($"XmlReader: {Environment.TickCount - t} ms");
		142	}
		143
118	void AssertRead(XmlReader reader, XmlNodeType expected) {	144	void AssertRead(XmlReader reader, XmlNodeType expected) {
119	Assert.IsTrue(reader.Read());	145	Assert.IsTrue(reader.Read());
120	Console.WriteLine($"{new string(' ', reader.Depth*2)}{reader}");	146	Console.WriteLine($"{new string(' ', reader.Depth * 2)}{reader}");
121	Assert.AreEqual(expected, reader.NodeType);	147	Assert.AreEqual(expected, reader.NodeType);
122	}	148	}
123		149

Implab.Playground/Program.cs +7 -97

@@ -78,103 +78,13 namespace Implab.Playground {
78		78
79	static void Main(string[] args) {	79	static void Main(string[] args) {
80		80
81	~~//var queue = new ConcurrentQueue<int>()~~;	81	var t = Environment.TickCount;
82	var queue = new AsyncQueue<int>();	82	using (var reader = JsonReader.Create("e:\\citylots.json")) {
83	//var queue = new SimpleAsyncQueue<int>();	83	while (reader.Read()) {
84		84	}
85	const int wBatch = 32;	85	}
86	const long wCount = 1000000;	86
87	const long total = wBatch * wCount * 3;	87	Console.WriteLine($"JsonReader: {Environment.TickCount - t} ms");
88
89	long r1 = 0, r2 = 0, r3 = 0;
90	const int rBatch = 1000;
91	long read = 0;
92
93	var t1 = Environment.TickCount;
94
95	AsyncPool.RunThread(
96	() => {
97	var buffer = new int[wBatch];
98	for (int i = 0; i < wBatch; i++)
99	buffer[i] = 1;
100
101	for (int i = 0; i < wCount; i++)
102	EnqueueRange(queue, buffer, 0, wBatch);
103	Console.WriteLine("done writer #1: {0} ms", Environment.TickCount - t1);
104	},
105	() => {
106	var buffer = new int[wBatch];
107	for (int i = 0; i < wBatch; i++)
108	buffer[i] = 1;
109
110	for (int i = 0; i < wCount; i++)
111	EnqueueRange(queue, buffer, 0, wBatch);
112	Console.WriteLine("done writer #2: {0} ms", Environment.TickCount - t1);
113	},
114	() => {
115	var buffer = new int[wBatch];
116	for (int i = 0; i < wBatch; i++)
117	buffer[i] = 1;
118
119	for (int i = 0; i < wCount; i++)
120	EnqueueRange(queue, buffer, 0, wBatch);
121	Console.WriteLine("done writer #3: {0} ms", Environment.TickCount - t1);
122	},
123	() => {
124	var buffer = new int[rBatch];
125
126	while (read < total) {
127	int actual;
128	if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
129	for (int i = 0; i < actual; i++)
130	r1 += buffer[i];
131	Interlocked.Add(ref read, actual);
132	}
133	}
134
135	Console.WriteLine("done reader #1: {0} ms", Environment.TickCount - t1);
136	}/*,
137	() => {
138	var buffer = new int[rBatch];
139
140	while (read < total) {
141	int actual;
142	if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
143	for (int i = 0; i < actual; i++)
144	r2 += buffer[i];
145	Interlocked.Add(ref read, actual);
146	}
147	}
148
149	Console.WriteLine("done reader #2: {0} ms", Environment.TickCount - t1);
150	}//,
151	() => {
152	var buffer = new int[rBatch];
153
154	while (read < total) {
155	int actual;
156	if (TryDequeueRange(queue, buffer, 0, rBatch, out actual)) {
157	for (int i = 0; i < actual; i++)
158	r3 += buffer[i];
159	Interlocked.Add(ref read, actual);
160	}
161	}
162
163	Console.WriteLine("done reader #3: {0} ms", Environment.TickCount - t1);
164	}*/
165	)
166	.PromiseAll()
167	.Join();
168
169
170	Console.WriteLine(
171	"done: {0} ms, summ#1: {1}, summ#2: {2}, total: {3}, count: {4}",
172	Environment.TickCount - t1,
173	r1,
174	r2,
175	r1 + r2 + r3,
176	total
177	);
178		88
179	Console.WriteLine("done");	89	Console.WriteLine("done");
180	}	90	}

Implab/Automaton/AutomatonConst.cs +2 -2

             namespace Implab.Automaton {
                 public static class AutomatonConst {
-                    public const int UNREACHABLE_STATE = -1;
+                    public const int UnreachableState = -1;
-                    public const int UNCLASSIFIED_INPUT = 0;
+                    public const int UnclassifiedInput = 0;
                 }
             }

Implab/Automaton/DFATable.cs +4 -4

                         for (int i = 0; i < StateCount; i++)
                             for (int j = 0; j < AlphabetSize; j++)
-                                table[i, j] = AutomatonConst.UNREACHABLE_STATE;
+                                table[i, j] = AutomatonConst.UnreachableState;
                         foreach (var t in this)
                             table[t.s1,t.edge] = (byte)t.s2;
                         var nextCls = 0;
                         foreach (var item in minClasses) {
-                            if (nextCls == AutomatonConst.UNCLASSIFIED_INPUT)
+                            if (nextCls == AutomatonConst.UnclassifiedInput)
                                 nextCls++;
                             // сохраняем DFAConst.UNCLASSIFIED_INPUT
-                            var cls = item.Contains(AutomatonConst.UNCLASSIFIED_INPUT) ? AutomatonConst.UNCLASSIFIED_INPUT : nextCls++;
+                            var cls = item.Contains(AutomatonConst.UnclassifiedInput) ? AutomatonConst.UnclassifiedInput : nextCls++;
                             optimalDFA.AddSymbol(cls);
                             foreach (var a in item)
                             data.Add(String.Format(
                                 "{0} -> {2} [label={1}];",
                                 String.Join("", stateAlphabet.GetSymbols(t.s1)),
-                                ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UNCLASSIFIED_INPUT ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
+                                ToLiteral(ToLiteral(String.Join("", t.edge == AutomatonConst.UnclassifiedInput ? new [] { "@" } : inputAlphabet.GetSymbols(t.edge).Select(x => x.ToString())))),
                                 String.Join("", stateAlphabet.GetSymbols(t.s2))
                             ));
                         data.Add("}");

Implab/Automaton/MapAlphabet.cs +1 -1

                             return cls;
                         if (!m_supportUnclassified)
                             throw new ArgumentOutOfRangeException("symbol", "The specified symbol isn't in the alphabet");
-                        return AutomatonConst.UNCLASSIFIED_INPUT;
+                        return AutomatonConst.UnclassifiedInput;
                     }
                     public int Count {

Implab/Automaton/RegularExpressions/RegularExpressionVisitor.cs +1 -1

                         if (m_root == null)
                             m_root = token;
                         m_idx++;
-                        m_indexes[m_idx] = AutomatonConst.UNCLASSIFIED_INPUT;
+                        m_indexes[m_idx] = AutomatonConst.UnclassifiedInput;
                         m_firstpos = new HashSet<int>(new[] { m_idx });
                         m_lastpos = new HashSet<int>(new[] { m_idx });
                         Followpos(m_idx);

Implab/Formats/CharAlphabet.cs +1 -1

             using System;
             namespace Implab.Formats {
-                public class CharAlphabet: IndexedAlphabetBase<char> {
+                public class CharAlphabet : IndexedAlphabetBase<char> {
                     public override int GetSymbolIndex(char symbol) {
                         return symbol;

Implab/Formats/CharMap.cs +2 -2

                     }
                     public bool Contains(char symbol) {
-                        return symbol >= m_min && symbol <= m_max && m_map[symbol-m_min] != AutomatonConst.UNCLASSIFIED_INPUT;
+                        return symbol >= m_min && symbol <= m_max && m_map[symbol-m_min] != AutomatonConst.UnclassifiedInput;
                     }
                     public IEnumerable<char> GetSymbols(int cls) {
                     [MethodImpl(MethodImplOptions.AggressiveInlining)]
                     public int Translate(char symbol) {
-                        return symbol >= m_min && symbol <= m_max ? m_map[symbol-m_min] : AutomatonConst.UNCLASSIFIED_INPUT;
+                        return symbol >= m_min && symbol <= m_max ? m_map[symbol-m_min] : AutomatonConst.UnclassifiedInput;
                     }
                 }
             }

Implab/Formats/Grammar.cs +3 -3

                     }
                     protected SymbolToken UnclassifiedToken() {
-                        return new SymbolToken(AutomatonConst.UNCLASSIFIED_INPUT);
+                        return new SymbolToken(AutomatonConst.UnclassifiedInput);
                     }
                     protected void DefineAlphabet(IEnumerable<TSymbol> alphabet) {
                     int TranslateOrAdd(TSymbol ch) {
                         var t = AlphabetBuilder.Translate(ch);
-                        if (t == AutomatonConst.UNCLASSIFIED_INPUT)
+                        if (t == AutomatonConst.UnclassifiedInput)
                             t = AlphabetBuilder.DefineSymbol(ch);
                         return t;
                     }
                     int TranslateOrDie(TSymbol ch) {
                         var t = AlphabetBuilder.Translate(ch);
-                        if (t == AutomatonConst.UNCLASSIFIED_INPUT)
+                        if (t == AutomatonConst.UnclassifiedInput)
                                 throw new ApplicationException(String.Format("Symbol '{0}' is UNCLASSIFIED", ch));
                         return t;
                     }

Implab/Formats/InputScanner.cs +1 -1

                         while(offset < max) {
                             next = m_dfa[next, m_alphabet.Translate(data[offset])];
-                            if (next == AutomatonConst.UNREACHABLE_STATE) {
+                            if (next == AutomatonConst.UnreachableState) {
                                 // scanner stops on the next position after last recognized symbol
                                 m_position = offset;
                                 return false;

Implab/Formats/Json/JsonGrammar.cs +9 -9

                         get { return _instance.Value; }
                     }
-                    readonly InputScanner<TokenType> m_jsonExpression;
+                    readonly FastInputScanner<TokenType> m_jsonExpression;
-                    readonly InputScanner<TokenType> m_stringExpression;
+                    readonly FastInputScanner<TokenType> m_stringExpression;
                     readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
                     public CharAlphabet DefaultAlphabet { get { return m_defaultAlphabet; } }
                             .Or(unescaped.Closure().Tag(TokenType.UnescapedChar));
-                        m_jsonExpression = BuildScanner(jsonExpression);
+                        m_jsonExpression = BuildFastScanner(jsonExpression);
-                        m_stringExpression = BuildScanner(jsonStringExpression);
+                        m_stringExpression = BuildFastScanner(jsonStringExpression);
                     }
-                    public static InputScanner<TokenType> CreateJsonExpressionScanner() {
+                    public static FastInputScanner<TokenType> CreateJsonExpressionScanner() {
                         return Instance.m_jsonExpression.Clone();
                     }
-                    public static InputScanner<TokenType> CreateStringExpressionScanner() {
+                    public static FastInputScanner<TokenType> CreateStringExpressionScanner() {
                         return Instance.m_stringExpression.Clone();
                     }
                         return SymbolToken(Enumerable.Range(start, stop - start + 1).Select(x => (char)x));
                     }
-                    public InputScanner<TokenType> BuildScanner(Token regexp) {
+                    public FastInputScanner<TokenType> BuildFastScanner(Token regexp) {
                         var dfa = new RegularDFA<char, TokenType>(AlphabetBuilder);
                         var visitor = new RegularExpressionVisitor<TokenType>(dfa);
                         var ab = new CharAlphabet();
                         var optimal = dfa.Optimize(ab);
-                        return new InputScanner<TokenType>(
+                        return new FastInputScanner<TokenType>(
                             optimal.CreateTransitionTable(),
                             optimal.CreateFinalStateTable(),
                             NormalizeTags(optimal.CreateTagTable()),
                             optimal.InitialState,
-                            ab.CreateCharMap()
+                            ab.GetTranslationMap()
                         );
                     }

Implab/Formats/Json/JsonReader.cs +5 -5

                         public bool Move(JsonTokenType token) {
                             var next = m_dfa[m_state, (int)token];
-                            if (next == AutomatonConst.UNREACHABLE_STATE)
+                            if (next == AutomatonConst.UnreachableState)
                                 return false;
                             m_state = next;
                             return true;
                     MemberContext m_memberContext = MemberContext.MemberValue;
                     JsonElementType m_elementType;
-                    object m_elementValue;
+                    string m_elementValue;
                     string m_memberName = String.Empty;
                     Stack<ParserContext> m_stack = new Stack<ParserContext>();
                     /// <summary>
                     /// Значение элемента. Только для элементов типа <see cref="JsonElementType.Value"/>, для остальных <c>null</c>
                     /// </summary>
-                    public object ElementValue {
+                    public string ElementValue {
                         get { return m_elementValue; }
                     }
                                     return true;
                                 case JsonTokenType.Number:
                                     m_elementType = JsonElementType.Value;
-                                    m_elementValue = double.Parse(tokenValue, CultureInfo.InvariantCulture);
+                                    m_elementValue = tokenValue;
                                     return true;
                                 case JsonTokenType.Literal:
                                     m_elementType = JsonElementType.Value;
-                                    m_elementValue = ParseLiteral(tokenValue);
+                                    m_elementValue = tokenValue == "null" ? null : tokenValue;
                                     return true;
                                 case JsonTokenType.NameSeparator:
                                     m_memberContext = MemberContext.MemberValue;

Implab/Formats/Json/JsonScanner.cs +5 -5

                 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
                 /// </summary>
                 public abstract class JsonScanner : Disposable {
-                    readonly InputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
+                    readonly FastInputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
-                    readonly InputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
+                    readonly FastInputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
                     readonly char[] m_unescapeBuf = new char[4];
                     readonly char[] m_buffer;
                         m_length = length;
                     }
-                    bool ReadChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
+                    bool ReadChunk(FastInputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
                         scanner.ResetState();
                         while(scanner.Scan(m_buffer, m_pos, m_length)) {
                         return true;
                     }
-                    bool ReadStringChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
+                    bool ReadStringChunk(FastInputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
                         scanner.ResetState();
                         while (scanner.Scan(m_buffer, m_pos, m_length)) {
                         // scanner stops as scannerPos
                         if (!scanner.IsFinal)
-                            throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
+                            throw new ParserException($"Unexpected character '{m_buffer[scannerPos]}'");
                         if (scannerPos != m_pos) {
                             m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);

Implab/Implab.csproj +1 0

                 <Compile Include="Diagnostics\TraceEventType.cs" />
                 <Compile Include="Diagnostics\TraceSourceAttribute.cs" />
                 <Compile Include="Formats\CharMap.cs" />
+                <Compile Include="Formats\FastInpurScanner.cs" />
                 <Compile Include="Formats\InputScanner.cs" />
                 <Compile Include="Formats\Json\JsonStringScanner.cs" />
                 <Compile Include="Formats\Json\JsonTextScanner.cs" />

Implab/Xml/JsonXmlReader.cs +11 -27

                     int m_xmlDepth;
                     XmlSimpleAttribute[] m_attributes;
-                    object m_value;
+                    string m_value;
                     bool m_isEmpty;
                     XmlNodeType m_nodeType = XmlNodeType.None;
                     public override string Value {
                         get {
-                            return ConvertValueToString(m_value);
+                            return m_value;
                         }
                     }
-                    static string ConvertValueToString(object value) {
-                        if (value == null)
-                            return string.Empty;
-                        switch (Convert.GetTypeCode(value)) {
-                            case TypeCode.Double:
-                                return ((double)value).ToString(CultureInfo.InvariantCulture);
-                            case TypeCode.String:
-                                return (string)value;
-                            case TypeCode.Boolean:
-                                return (bool)value ? "true" : "false";
-                            default:
-                                return value.ToString();
                     public override string GetAttribute(int i) {
                         Safe.ArgumentInRange(i, 0, AttributeCount - 1, nameof(i));
-                        return ConvertValueToString(m_attributes[i].Value);
+                        return m_attributes[i].Value;
                     }
                     public override string GetAttribute(string name) {
                             return null;
                         var qName = m_context.Resolve(name);
                         var attr = Array.Find(m_attributes, x => x.QName == qName);
-                        var value = ConvertValueToString(attr?.Value);
+                        var value = attr?.Value;
                         return value == string.Empty ? null : value;
                     }
                             return null;
                         var qName = new XmlQualifiedName(name, namespaceURI);
                         var attr = Array.Find(m_attributes, x => x.QName == qName);
-                        var value = ConvertValueToString(attr?.Value);
+                        var value = attr?.Value;
                         return value == string.Empty ? null : value;
                     }
                         }
                     }
-                    void ValueNode(object value) {
+                    void ValueNode(string value) {
                         if (!IsSibling()) // the node is nested
                             m_xmlDepth++;
                                 if (attr.QName.Name == "xmlns") {
                                     if (context == m_context)
                                          context = new XmlNameContext(m_context, m_xmlDepth);
-                                    context.DefinePrefix(ConvertValueToString(attr.Value), string.Empty);
+                                    context.DefinePrefix(attr.Value, string.Empty);
                                 } else if (attr.Prefix == m_xmlnsPrefix) {
                                     if (context == m_context)
                                         context = new XmlNameContext(m_context, m_xmlDepth);
-                                    context.DefinePrefix(ConvertValueToString(attr.Value), attr.QName.Name);
+                                    context.DefinePrefix(attr.Value, attr.QName.Name);
                                 } else {
                                     string attrPrefix;
                                     if (string.IsNullOrEmpty(attr.QName.Namespace))
                                                 m_jsonValueName,
                                                 m_jsonNamespace,
                                                 new[] {
-                                                    new XmlSimpleAttribute("nil", m_xsiNamespace, m_xsiPrefix, true)
+                                                    new XmlSimpleAttribute("nil", m_xsiNamespace, m_xsiPrefix, "true")
                                                 },
                                                 true
                                             );
                     public override string ToString() {
                         switch (NodeType) {
                             case XmlNodeType.Element:
-                                return $"<{Name} {string.Join(" ", (m_attributes ?? new XmlSimpleAttribute[0]).Select(x => $"{x.Prefix}{(string.IsNullOrEmpty(x.Prefix) ? "" : ":")}{x.QName.Name}='{ConvertValueToString(x.Value)}'"))} {(IsEmptyElement ? "/" : "")}>";
+                                return $"<{Name} {string.Join(" ", (m_attributes ?? new XmlSimpleAttribute[0]).Select(x => $"{x.Prefix}{(string.IsNullOrEmpty(x.Prefix) ? "" : ":")}{x.QName.Name}='{x.Value}'"))} {(IsEmptyElement ? "/" : "")}>";
                             case XmlNodeType.Attribute:
                                 return $"@{Name}";
                             case XmlNodeType.Text:

Implab/Xml/XmlSimpleAttribute.cs +2 -2

             namespace Implab.Xml {
                 public class XmlSimpleAttribute {
-                    public XmlSimpleAttribute(string name, string ns, string prefix, object value) {
+                    public XmlSimpleAttribute(string name, string ns, string prefix, string value) {
                         QName = new XmlQualifiedName(name, ns);
                         Prefix = prefix;
                         Value = value;
                     public string Prefix { get; set; }
-                    public object Value { get; set; }
+                    public string Value { get; set; }
                 }
             }

General Comments 3

sergey

|

Auto status change to "Under Review"

sergey

|

ok, latest stable version should be in default

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages