diff --git a/ln.parse.sln b/ln.parse.sln
new file mode 100644
index 0000000..d519caa
--- /dev/null
+++ b/ln.parse.sln
@@ -0,0 +1,48 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26124.0
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.parse", "ln.parse\ln.parse.csproj", "{CFAE942E-F3A0-43B2-A579-FAB00A7B1883}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.parse.tests", "ln.parse.tests\ln.parse.tests.csproj", "{1E2ED52D-5350-46D9-AEA9-ABC892107548}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Debug|x64 = Debug|x64
+ Debug|x86 = Debug|x86
+ Release|Any CPU = Release|Any CPU
+ Release|x64 = Release|x64
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x64.Build.0 = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x86.Build.0 = Debug|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|Any CPU.Build.0 = Release|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x64.ActiveCfg = Release|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x64.Build.0 = Release|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x86.ActiveCfg = Release|Any CPU
+ {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x86.Build.0 = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x64.Build.0 = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x86.Build.0 = Debug|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|Any CPU.Build.0 = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x64.ActiveCfg = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x64.Build.0 = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x86.ActiveCfg = Release|Any CPU
+ {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x86.Build.0 = Release|Any CPU
+ EndGlobalSection
+EndGlobal
diff --git a/ln.parse.tests/TokenizerTests.cs b/ln.parse.tests/TokenizerTests.cs
new file mode 100644
index 0000000..4f34d88
--- /dev/null
+++ b/ln.parse.tests/TokenizerTests.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Reflection;
+using ln.parse.tokenizer;
+using NUnit.Framework;
+
+namespace ln.parse.tests
+{
+ public class TokenizerTests
+ {
+ Tokenizer tokenizer;
+
+ [SetUp]
+ public void Setup()
+ {
+ tokenizer = Tokenizer.CreateDefaultTokenizer();
+ }
+
+ [Test]
+ public void Test_Integer()
+ {
+ Token[] token = tokenizer.Parse("654372");
+
+ TestContext.Out.WriteLine("Tokens: {0}", token);
+
+ Assert.AreEqual(1, token.Length);
+ Assert.IsTrue(token[0] is Token.IntegerToken);
+ Assert.AreEqual("654372", token[0].Value);
+
+ Assert.Pass();
+ }
+
+ [Test]
+ public void Test_Float()
+ {
+ Token[] token = tokenizer.Parse("654372.3524");
+
+ TestContext.Out.WriteLine("Tokens: {0}", token);
+
+ Assert.AreEqual(1, token.Length);
+ Assert.IsTrue(token[0] is Token.FloatToken);
+ Assert.AreEqual("654372.3524", token[0].Value);
+
+ Assert.Pass();
+ }
+
+ [Test]
+ public void Test_String()
+ {
+ Token[] token = tokenizer.Parse("\"Hallo Welt, ich bin ein \\\"String\\\"!\"");
+
+ TestContext.Out.WriteLine("Tokens: {0}", token);
+
+ Assert.AreEqual(1, token.Length);
+ Assert.IsTrue(token[0] is Token.StringToken);
+ Assert.AreEqual("\"Hallo Welt, ich bin ein \\\"String\\\"!\"", token[0].Value);
+
+ Assert.Pass();
+ }
+ }
+}
\ No newline at end of file
diff --git a/ln.parse.tests/ln.parse.tests.csproj b/ln.parse.tests/ln.parse.tests.csproj
new file mode 100644
index 0000000..4450e03
--- /dev/null
+++ b/ln.parse.tests/ln.parse.tests.csproj
@@ -0,0 +1,19 @@
+
+
+
+ netcoreapp3.1
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ln.parse.csproj b/ln.parse/ln.parse.csproj
similarity index 58%
rename from ln.parse.csproj
rename to ln.parse/ln.parse.csproj
index d1c7cd1..c1679f0 100644
--- a/ln.parse.csproj
+++ b/ln.parse/ln.parse.csproj
@@ -2,14 +2,13 @@
netcoreapp3.1
+ 0.0.1
+ Harald Wolff-Thobaben
+ l--n.de
+
-
-
-
-
-
diff --git a/tokenizer/RegularExpressionMatcher.cs b/ln.parse/tokenizer/RegularExpressionMatcher.cs
similarity index 100%
rename from tokenizer/RegularExpressionMatcher.cs
rename to ln.parse/tokenizer/RegularExpressionMatcher.cs
diff --git a/tokenizer/SourceBuffer.cs b/ln.parse/tokenizer/SourceBuffer.cs
similarity index 100%
rename from tokenizer/SourceBuffer.cs
rename to ln.parse/tokenizer/SourceBuffer.cs
diff --git a/tokenizer/Token.cs b/ln.parse/tokenizer/Token.cs
similarity index 100%
rename from tokenizer/Token.cs
rename to ln.parse/tokenizer/Token.cs
diff --git a/tokenizer/TokenMatcher.cs b/ln.parse/tokenizer/TokenMatcher.cs
similarity index 92%
rename from tokenizer/TokenMatcher.cs
rename to ln.parse/tokenizer/TokenMatcher.cs
index cc4e46b..ad00da8 100644
--- a/tokenizer/TokenMatcher.cs
+++ b/ln.parse/tokenizer/TokenMatcher.cs
@@ -16,7 +16,7 @@ namespace ln.parse.tokenizer
public abstract bool Match(SourceBuffer sourceBuffer, out Token token);
public static readonly TokenMatcher INTEGER = new RegularExpressionMatcher("^-?\\d+", (SourceBuffer sourceBuffer, int start, int length) => new Token.IntegerToken(sourceBuffer, start, length));
- public static readonly TokenMatcher FLOAT = new RegularExpressionMatcher("^-?\\d+.\\d*", (SourceBuffer sourceBuffer, int start, int length) => new Token.FloatToken(sourceBuffer, start, length));
+ public static readonly TokenMatcher FLOAT = new RegularExpressionMatcher("^-?\\d+\\.\\d*", (SourceBuffer sourceBuffer, int start, int length) => new Token.FloatToken(sourceBuffer, start, length));
public static readonly TokenMatcher STRING = new RegularExpressionMatcher("^\\\"(\\\\\"|.)*?\\\"", (SourceBuffer sourceBuffer, int start, int length) => new Token.StringToken(sourceBuffer, start, length));
public static readonly TokenMatcher IDENTIFIER = new RegularExpressionMatcher("^\\w][a-zA-Z0-9_]*", (SourceBuffer sourceBuffer, int start, int length) => new Token.IdentifierToken(sourceBuffer, start, length));
public static readonly TokenMatcher OPERATOR = new RegularExpressionMatcher("\\+|\\-|\\*|\\/|\\||\\&|\\|\\||\\&\\&", (SourceBuffer sourceBuffer, int start, int length) => new Token.OperatorToken(sourceBuffer, start, length));
diff --git a/tokenizer/Tokenizer.cs b/ln.parse/tokenizer/Tokenizer.cs
similarity index 75%
rename from tokenizer/Tokenizer.cs
rename to ln.parse/tokenizer/Tokenizer.cs
index 81f08a2..82381fd 100644
--- a/tokenizer/Tokenizer.cs
+++ b/ln.parse/tokenizer/Tokenizer.cs
@@ -35,11 +35,20 @@ namespace ln.parse.tokenizer
throw new FormatException(String.Format("invalid token at {0}",sourceBuffer.TextPosition));
tokens.Add(token);
+ sourceBuffer.LinearPosition += token.Length;
}
return tokens.ToArray();
}
+ public static Tokenizer CreateDefaultTokenizer() =>
+ new Tokenizer()
+ .Add(TokenMatcher.WHITESPACE)
+ .Add(TokenMatcher.FLOAT)
+ .Add(TokenMatcher.INTEGER)
+ .Add(TokenMatcher.STRING)
+ .Add(TokenMatcher.OPERATOR)
+ .Add(TokenMatcher.BRACKET);
}
}