IToken

IToken is interface for token types.

/// <summary>Token features</summary>
public interface IToken
{
    /// <summary>Memory source</summary>
    ReadOnlyMemory<char> Memory { get; set; }
    /// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
    IToken[] Children { get; set; }
    /// <summary>Accept visitor.</summary>
    bool Accept(ITokenVisitor visitor);
}

/// <summary>Token visitor base interface.</summary>
public interface ITokenVisitor { }
/// <summary>Token visitor for <typeparamref name="T"/>.</summary>
public interface ITokenVisitor<T> : ITokenVisitor where T : IToken
{
    /// <summary>Visit token</summary>
    void Visit(T token);
}

Token type indicates the role of a character segment.

// The tokenized text
ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
// Create composite token manually, for purpose of example
CompositeToken compositeToken = new CompositeToken
{
    Memory = text,
    Children = new IToken[]
    {
        new DecimalToken { Memory = text.Slice(0,1) },
        new WhitespaceToken { Memory = text.Slice(1,1) },
        new OperandToken { Memory = text.Slice(2, 1) },
        new WhitespaceToken { Memory = text.Slice(3,1) },
        new DecimalToken { Memory = text.Slice(4,1) },
        new WhitespaceToken { Memory = text.Slice(5,1) },
        new OperandToken { Memory = text.Slice(6, 1) },
        new WhitespaceToken { Memory = text.Slice(7,1) },
        new DecimalToken { Memory = text.Slice(8,1) },
    }
};

.PrintTree() extension method prints structural token as a tree.

WriteLine(compositeToken.PrintTree());

CompositeToken:  "1 + 2 = 3"
├── DecimalToken:  "1"
├── WhitespaceToken:  " "
├── OperandToken:  "+"
├── WhitespaceToken:  " "
├── DecimalToken:  "2"
├── WhitespaceToken:  " "
├── OperandToken:  "="
├── WhitespaceToken:  " "
└── DecimalToken:  "3"

.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong) prints with indices.

WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));

[0:9] CompositeToken:  "1 + 2 = 3"
├── [0:1] DecimalToken:  "1"
├── [1:2] WhitespaceToken:  " "
├── [2:3] OperandToken:  "+"
├── [3:4] WhitespaceToken:  " "
├── [4:5] DecimalToken:  "2"
├── [5:6] WhitespaceToken:  " "
├── [6:7] OperandToken:  "="
├── [7:8] WhitespaceToken:  " "
└── [8:9] DecimalToken:  "3"

.VisitTree() visits each tree node.

foreach (var line in compositeToken.VisitTree())
    WriteLine(line);

.Text() gets original string or creates a new string if represents a slice.

string text_ = compositeToken.Text();

.Memory.Index() and .Memory.Length return the spanned memory range.

int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;

.Accept(visitor) takes a visitor.

compositeToken.Accept(new TokenVisitor());

public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
{
    public void Visit(CompositeToken token)
    {
        foreach (IToken child in token.Children)
            child.Accept(this);
    }
    public void Visit(DecimalToken token) => Console.WriteLine(token);
    public void Visit(OperandToken token) => Console.WriteLine(token);
}

[0:1] DecimalToken "1"
[2:3] OperandToken "+"
[4:5] DecimalToken "2"
[6:7] OperandToken "="
[8:9] DecimalToken "3"

Implementation

Implementation holds a ReadOnlyMemory<char> and optinally a child array.

using Avalanche.Tokenizer;
using Avalanche.Utilities;

/// <summary>Example token.</summary>
public struct IdentifierToken : IToken
{
    /// <summary>Create token</summary>
    public IdentifierToken() { }
    /// <summary>Text source</summary>
    public ReadOnlyMemory<char> Memory { get; set; } = default;
    /// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
    public IToken[] Children { get; set; } = Array.Empty<IToken>();
    /// <summary>Accept visitor.</summary>
    public bool Accept(ITokenVisitor visitor) { if (visitor is ITokenVisitor<IdentifierToken> c) { c.Visit(this); return true; } else return false; }
    /// <summary>Print token as string</summary>
    public override string ToString() { int index = Memory.Index(); return $"[{index}:{index + Memory.Length}] {GetType().Name} \"{Memory}\""; }
}

The following tokens are introduced in Avalanche.Tokenizer.

Token	Description
CompositeToken	Structural token
DecimalToken	Decimal number
KeyToken	Key part
KeyValueToken	Key-value
ValueToken	Value part
MalformedToken	Malformed formatting
NameToken	Name
NewLineToken	Indicates '\n'
OperandToken	Operand, such as '%', '+', etc
ParenthesisToken	'(' or ')'
RangeToken	Range, e.g. "0..2"
SeparatorToken	Separator, e.g. ','
TextToken	Text part
VariableToken	Variable name
WhitespaceToken	Whitespace part
GroupToken	Group
GroupSeparatorToken	Group separator

Full Example

Full example

using System.Globalization;
using Avalanche.Tokenizer;
using Avalanche.Utilities;
using static System.Console;

public class token
{
    public static void Run()
    {
        {
            // <01>
            // The tokenized text
            ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
            // Create composite token manually, for purpose of example
            CompositeToken compositeToken = new CompositeToken
            {
                Memory = text,
                Children = new IToken[]
                {
                    new DecimalToken { Memory = text.Slice(0,1) },
                    new WhitespaceToken { Memory = text.Slice(1,1) },
                    new OperandToken { Memory = text.Slice(2, 1) },
                    new WhitespaceToken { Memory = text.Slice(3,1) },
                    new DecimalToken { Memory = text.Slice(4,1) },
                    new WhitespaceToken { Memory = text.Slice(5,1) },
                    new OperandToken { Memory = text.Slice(6, 1) },
                    new WhitespaceToken { Memory = text.Slice(7,1) },
                    new DecimalToken { Memory = text.Slice(8,1) },
                }
            };
            // </01>
            // <02>
            WriteLine(compositeToken.PrintTree());
            // </02>
            // <02B>
            WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            // </02B>
            // <03>
            foreach (var line in compositeToken.VisitTree())
                WriteLine(line);
            // </03>
            // <04>
            string text_ = compositeToken.Text();
            // </04>
            // <05>
            int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;
            // </05>
            // <06>
            compositeToken.Accept(new TokenVisitor());
            // </06>
        }
    }

    // <99>
    public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
    {
        public void Visit(CompositeToken token)
        {
            foreach (IToken child in token.Children)
                child.Accept(this);
        }
        public void Visit(DecimalToken token) => Console.WriteLine(token);
        public void Visit(OperandToken token) => Console.WriteLine(token);
    }
    // </99>
}