From b46386560b03a3ba9e81fe8fb28b00274ff7d5cf Mon Sep 17 00:00:00 2001 From: Harald Wolff Date: Fri, 20 Nov 2020 18:45:39 +0100 Subject: [PATCH] Reorganize repository to use .sln --- ln.parse.sln | 48 +++++++++++++++ ln.parse.tests/TokenizerTests.cs | 60 +++++++++++++++++++ ln.parse.tests/ln.parse.tests.csproj | 19 ++++++ ln.parse.csproj => ln.parse/ln.parse.csproj | 9 ++- .../tokenizer}/RegularExpressionMatcher.cs | 0 .../tokenizer}/SourceBuffer.cs | 0 {tokenizer => ln.parse/tokenizer}/Token.cs | 0 .../tokenizer}/TokenMatcher.cs | 2 +- .../tokenizer}/Tokenizer.cs | 9 +++ 9 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 ln.parse.sln create mode 100644 ln.parse.tests/TokenizerTests.cs create mode 100644 ln.parse.tests/ln.parse.tests.csproj rename ln.parse.csproj => ln.parse/ln.parse.csproj (58%) rename {tokenizer => ln.parse/tokenizer}/RegularExpressionMatcher.cs (100%) rename {tokenizer => ln.parse/tokenizer}/SourceBuffer.cs (100%) rename {tokenizer => ln.parse/tokenizer}/Token.cs (100%) rename {tokenizer => ln.parse/tokenizer}/TokenMatcher.cs (92%) rename {tokenizer => ln.parse/tokenizer}/Tokenizer.cs (75%) diff --git a/ln.parse.sln b/ln.parse.sln new file mode 100644 index 0000000..d519caa --- /dev/null +++ b/ln.parse.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.parse", "ln.parse\ln.parse.csproj", "{CFAE942E-F3A0-43B2-A579-FAB00A7B1883}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ln.parse.tests", "ln.parse.tests\ln.parse.tests.csproj", "{1E2ED52D-5350-46D9-AEA9-ABC892107548}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x64.ActiveCfg = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x64.Build.0 = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x86.ActiveCfg = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Debug|x86.Build.0 = Debug|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|Any CPU.Build.0 = Release|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x64.ActiveCfg = Release|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x64.Build.0 = Release|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x86.ActiveCfg = Release|Any CPU + {CFAE942E-F3A0-43B2-A579-FAB00A7B1883}.Release|x86.Build.0 = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x64.ActiveCfg = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x64.Build.0 = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x86.ActiveCfg = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Debug|x86.Build.0 = Debug|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|Any CPU.Build.0 = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x64.ActiveCfg = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x64.Build.0 = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x86.ActiveCfg = Release|Any CPU + {1E2ED52D-5350-46D9-AEA9-ABC892107548}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/ln.parse.tests/TokenizerTests.cs b/ln.parse.tests/TokenizerTests.cs new file mode 100644 index 0000000..4f34d88 --- /dev/null +++ b/ln.parse.tests/TokenizerTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Reflection; +using ln.parse.tokenizer; +using NUnit.Framework; + +namespace ln.parse.tests +{ + public class TokenizerTests + { + Tokenizer tokenizer; + + [SetUp] + public void Setup() + { + tokenizer = Tokenizer.CreateDefaultTokenizer(); + } + + [Test] + public void Test_Integer() + { + Token[] token = tokenizer.Parse("654372"); + + TestContext.Out.WriteLine("Tokens: {0}", token); + + Assert.AreEqual(1, token.Length); + Assert.IsTrue(token[0] is Token.IntegerToken); + Assert.AreEqual("654372", token[0].Value); + + Assert.Pass(); + } + + [Test] + public void Test_Float() + { + Token[] token = tokenizer.Parse("654372.3524"); + + TestContext.Out.WriteLine("Tokens: {0}", token); + + Assert.AreEqual(1, token.Length); + Assert.IsTrue(token[0] is Token.FloatToken); + Assert.AreEqual("654372.3524", token[0].Value); + + Assert.Pass(); + } + + [Test] + public void Test_String() + { + Token[] token = tokenizer.Parse("\"Hallo Welt, ich bin ein \\\"String\\\"!\""); + + TestContext.Out.WriteLine("Tokens: {0}", token); + + Assert.AreEqual(1, token.Length); + Assert.IsTrue(token[0] is Token.StringToken); + Assert.AreEqual("\"Hallo Welt, ich bin ein \\\"String\\\"!\"", token[0].Value); + + Assert.Pass(); + } + } +} \ No newline at end of file diff --git a/ln.parse.tests/ln.parse.tests.csproj b/ln.parse.tests/ln.parse.tests.csproj new file mode 100644 index 0000000..4450e03 --- /dev/null +++ b/ln.parse.tests/ln.parse.tests.csproj @@ -0,0 +1,19 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + diff --git a/ln.parse.csproj b/ln.parse/ln.parse.csproj similarity index 58% rename from ln.parse.csproj rename to ln.parse/ln.parse.csproj index d1c7cd1..c1679f0 100644 --- a/ln.parse.csproj +++ b/ln.parse/ln.parse.csproj @@ -2,14 +2,13 @@ netcoreapp3.1 + 0.0.1 + Harald Wolff-Thobaben + l--n.de + - - - - - diff --git a/tokenizer/RegularExpressionMatcher.cs b/ln.parse/tokenizer/RegularExpressionMatcher.cs similarity index 100% rename from tokenizer/RegularExpressionMatcher.cs rename to ln.parse/tokenizer/RegularExpressionMatcher.cs diff --git a/tokenizer/SourceBuffer.cs b/ln.parse/tokenizer/SourceBuffer.cs similarity index 100% rename from tokenizer/SourceBuffer.cs rename to ln.parse/tokenizer/SourceBuffer.cs diff --git a/tokenizer/Token.cs b/ln.parse/tokenizer/Token.cs similarity index 100% rename from tokenizer/Token.cs rename to ln.parse/tokenizer/Token.cs diff --git a/tokenizer/TokenMatcher.cs b/ln.parse/tokenizer/TokenMatcher.cs similarity index 92% rename from tokenizer/TokenMatcher.cs rename to ln.parse/tokenizer/TokenMatcher.cs index cc4e46b..ad00da8 100644 --- a/tokenizer/TokenMatcher.cs +++ b/ln.parse/tokenizer/TokenMatcher.cs @@ -16,7 +16,7 @@ namespace ln.parse.tokenizer public abstract bool Match(SourceBuffer sourceBuffer, out Token token); public static readonly TokenMatcher INTEGER = new RegularExpressionMatcher("^-?\\d+", (SourceBuffer sourceBuffer, int start, int length) => new Token.IntegerToken(sourceBuffer, start, length)); - public static readonly TokenMatcher FLOAT = new RegularExpressionMatcher("^-?\\d+.\\d*", (SourceBuffer sourceBuffer, int start, int length) => new Token.FloatToken(sourceBuffer, start, length)); + public static readonly TokenMatcher FLOAT = new RegularExpressionMatcher("^-?\\d+\\.\\d*", (SourceBuffer sourceBuffer, int start, int length) => new Token.FloatToken(sourceBuffer, start, length)); public static readonly TokenMatcher STRING = new RegularExpressionMatcher("^\\\"(\\\\\"|.)*?\\\"", (SourceBuffer sourceBuffer, int start, int length) => new Token.StringToken(sourceBuffer, start, length)); public static readonly TokenMatcher IDENTIFIER = new RegularExpressionMatcher("^\\w][a-zA-Z0-9_]*", (SourceBuffer sourceBuffer, int start, int length) => new Token.IdentifierToken(sourceBuffer, start, length)); public static readonly TokenMatcher OPERATOR = new RegularExpressionMatcher("\\+|\\-|\\*|\\/|\\||\\&|\\|\\||\\&\\&", (SourceBuffer sourceBuffer, int start, int length) => new Token.OperatorToken(sourceBuffer, start, length)); diff --git a/tokenizer/Tokenizer.cs b/ln.parse/tokenizer/Tokenizer.cs similarity index 75% rename from tokenizer/Tokenizer.cs rename to ln.parse/tokenizer/Tokenizer.cs index 81f08a2..82381fd 100644 --- a/tokenizer/Tokenizer.cs +++ b/ln.parse/tokenizer/Tokenizer.cs @@ -35,11 +35,20 @@ namespace ln.parse.tokenizer throw new FormatException(String.Format("invalid token at {0}",sourceBuffer.TextPosition)); tokens.Add(token); + sourceBuffer.LinearPosition += token.Length; } return tokens.ToArray(); } + public static Tokenizer CreateDefaultTokenizer() => + new Tokenizer() + .Add(TokenMatcher.WHITESPACE) + .Add(TokenMatcher.FLOAT) + .Add(TokenMatcher.INTEGER) + .Add(TokenMatcher.STRING) + .Add(TokenMatcher.OPERATOR) + .Add(TokenMatcher.BRACKET); } }