ln.parse/ln.parse/tokenizer/Token.cs

using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Text;

namespace ln.parse.tokenizer
{
    public class Token
    {
        public SourceBuffer SourceBuffer { get; }
        public int LinearStart { get; }
        public int Length { get; }
        public string Value { get; private set; }

        public TextPosition TextPosition => SourceBuffer.GetTextPosition(LinearStart);

        public Token(SourceBuffer sourceBuffer, int start, int length)
        {
            SourceBuffer = sourceBuffer;
            LinearStart = start;
            Length = length;
        }
        public Token(SourceBuffer sourceBuffer, int start, int length, string value)
        {
            SourceBuffer = sourceBuffer;
            LinearStart = start;
            Length = length;
            Value = value;
        }

        public string TokenSource => SourceBuffer.GetText(LinearStart, Length);

        public virtual object NativeValue { get; }

        public bool Is<T>(String value) => (this is T) && Value.Equals(value);

        public class IntegerToken : Token
        {
        public IntegerToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Int32.Parse(Value);
        }
        public class FloatToken : Token
        {
            public FloatToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Double.Parse(Value);
        }
        public class StringToken : Token
        {
            public StringToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Value;
        }
        public class OperatorToken : Token
        {
            public OperatorToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Value[0];
        }
        public class WhiteSpaceToken : Token
        {
            public WhiteSpaceToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Value;
        }
        public class IdentifierToken : Token
        {
            public IdentifierToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Value;
        }
        public class BracketToken : Token
        {
            public BracketToken(SourceBuffer sourceBuffer, int start, int length, string value) : base(sourceBuffer, start, length, value) { }
            public override object NativeValue => Value[0];
        }
    }
}