IToken
IToken is interface for token types.
/// <summary>Token features</summary>
public interface IToken
{
/// <summary>Memory source</summary>
ReadOnlyMemory<char> Memory { get; set; }
/// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
IToken[] Children { get; set; }
/// <summary>Accept visitor.</summary>
bool Accept(ITokenVisitor visitor);
}
/// <summary>Token visitor base interface.</summary>
public interface ITokenVisitor { }
/// <summary>Token visitor for <typeparamref name="T"/>.</summary>
public interface ITokenVisitor<T> : ITokenVisitor where T : IToken
{
/// <summary>Visit token</summary>
void Visit(T token);
}
Token type indicates the role of a character segment.
// The tokenized text
ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
// Create composite token manually, for purpose of example
CompositeToken compositeToken = new CompositeToken
{
Memory = text,
Children = new IToken[]
{
new DecimalToken { Memory = text.Slice(0,1) },
new WhitespaceToken { Memory = text.Slice(1,1) },
new OperandToken { Memory = text.Slice(2, 1) },
new WhitespaceToken { Memory = text.Slice(3,1) },
new DecimalToken { Memory = text.Slice(4,1) },
new WhitespaceToken { Memory = text.Slice(5,1) },
new OperandToken { Memory = text.Slice(6, 1) },
new WhitespaceToken { Memory = text.Slice(7,1) },
new DecimalToken { Memory = text.Slice(8,1) },
}
};
.PrintTree() extension method prints structural token as a tree.
WriteLine(compositeToken.PrintTree());
CompositeToken: "1 + 2 = 3" ├── DecimalToken: "1" ├── WhitespaceToken: " " ├── OperandToken: "+" ├── WhitespaceToken: " " ├── DecimalToken: "2" ├── WhitespaceToken: " " ├── OperandToken: "=" ├── WhitespaceToken: " " └── DecimalToken: "3"
.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong) prints with indices.
WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
[0:9] CompositeToken: "1 + 2 = 3" ├── [0:1] DecimalToken: "1" ├── [1:2] WhitespaceToken: " " ├── [2:3] OperandToken: "+" ├── [3:4] WhitespaceToken: " " ├── [4:5] DecimalToken: "2" ├── [5:6] WhitespaceToken: " " ├── [6:7] OperandToken: "=" ├── [7:8] WhitespaceToken: " " └── [8:9] DecimalToken: "3"
.VisitTree() visits each tree node.
foreach (var line in compositeToken.VisitTree())
WriteLine(line);
.Text() gets original string or creates a new string if represents a slice.
string text_ = compositeToken.Text();
.Memory.Index() and .Memory.Length return the spanned memory range.
int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;
.Accept(visitor) takes a visitor.
compositeToken.Accept(new TokenVisitor());
public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
{
public void Visit(CompositeToken token)
{
foreach (IToken child in token.Children)
child.Accept(this);
}
public void Visit(DecimalToken token) => Console.WriteLine(token);
public void Visit(OperandToken token) => Console.WriteLine(token);
}
[0:1] DecimalToken "1" [2:3] OperandToken "+" [4:5] DecimalToken "2" [6:7] OperandToken "=" [8:9] DecimalToken "3"
Implementation
Implementation holds a ReadOnlyMemory<char> and optinally a child array.
using Avalanche.Tokenizer;
using Avalanche.Utilities;
/// <summary>Example token.</summary>
public struct IdentifierToken : IToken
{
/// <summary>Create token</summary>
public IdentifierToken() { }
/// <summary>Text source</summary>
public ReadOnlyMemory<char> Memory { get; set; } = default;
/// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
public IToken[] Children { get; set; } = Array.Empty<IToken>();
/// <summary>Accept visitor.</summary>
public bool Accept(ITokenVisitor visitor) { if (visitor is ITokenVisitor<IdentifierToken> c) { c.Visit(this); return true; } else return false; }
/// <summary>Print token as string</summary>
public override string ToString() { int index = Memory.Index(); return $"[{index}:{index + Memory.Length}] {GetType().Name} \"{Memory}\""; }
}
The following tokens are introduced in Avalanche.Tokenizer.
Token | Description |
---|---|
CompositeToken | Structural token |
DecimalToken | Decimal number |
KeyToken | Key part |
KeyValueToken | Key-value |
ValueToken | Value part |
MalformedToken | Malformed formatting |
NameToken | Name |
NewLineToken | Indicates '\n' |
OperandToken | Operand, such as '%', '+', etc |
ParenthesisToken | '(' or ')' |
RangeToken | Range, e.g. "0..2" |
SeparatorToken | Separator, e.g. ',' |
TextToken | Text part |
VariableToken | Variable name |
WhitespaceToken | Whitespace part |
GroupToken | Group |
GroupSeparatorToken | Group separator |
Full Example
Full example
using System.Globalization;
using Avalanche.Tokenizer;
using Avalanche.Utilities;
using static System.Console;
public class token
{
public static void Run()
{
{
// <01>
// The tokenized text
ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
// Create composite token manually, for purpose of example
CompositeToken compositeToken = new CompositeToken
{
Memory = text,
Children = new IToken[]
{
new DecimalToken { Memory = text.Slice(0,1) },
new WhitespaceToken { Memory = text.Slice(1,1) },
new OperandToken { Memory = text.Slice(2, 1) },
new WhitespaceToken { Memory = text.Slice(3,1) },
new DecimalToken { Memory = text.Slice(4,1) },
new WhitespaceToken { Memory = text.Slice(5,1) },
new OperandToken { Memory = text.Slice(6, 1) },
new WhitespaceToken { Memory = text.Slice(7,1) },
new DecimalToken { Memory = text.Slice(8,1) },
}
};
// </01>
// <02>
WriteLine(compositeToken.PrintTree());
// </02>
// <02B>
WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
// </02B>
// <03>
foreach (var line in compositeToken.VisitTree())
WriteLine(line);
// </03>
// <04>
string text_ = compositeToken.Text();
// </04>
// <05>
int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;
// </05>
// <06>
compositeToken.Accept(new TokenVisitor());
// </06>
}
}
// <99>
public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
{
public void Visit(CompositeToken token)
{
foreach (IToken child in token.Children)
child.Accept(this);
}
public void Visit(DecimalToken token) => Console.WriteLine(token);
public void Visit(OperandToken token) => Console.WriteLine(token);
}
// </99>
}