• Avalanche.Core
Search Results for

    Show / Hide Table of Contents
    • Avalanche.Accessor
      • Introduction
      • IAccessor
        • IAccessor
        • IListAccessor
        • IMapAccessor
        • IRecordAccessor
        • IContentAccessor
        • IOneOfAccessor
        • IAnyAccessor
      • .Net
        • Introduction
        • IList<T>
        • IDictionary<K,V>
        • FieldInfo
        • OneOfAttribute
        • StructLayoutAttribute
        • Class
      • Protobuf
        • Introduction
      • Articles
        • Dependency Injection
        • AccessorMessages
    • Avalanche.Binding
      • Introduction
    • Avalanche.Core
      • License
    • Avalanche.DataType
      • Introduction
      • DataType
        • IDataType
        • IListType
        • IMapType
        • IRecordType
        • IFieldType
        • IOneOfType
        • IAnyType
        • IStringType
        • IValueType
        • IIntegerType
        • IEnumerationType
        • IRealType
      • .Net
        • Introduction
        • IList<T>
        • IDictionary<K,V>
        • FieldInfo
        • Enum
        • OneOfAttribute
        • StructLayoutAttribute
        • Class
      • Protobuf
        • Introduction
      • Articles
        • DataTypeRequest
        • PrintTree
        • DataTypeMessages
    • Avalanche.Emit
      • Introduction
      • TypeBuilder
      • ConstructorBuilder
      • MethodBuilder
      • PropertyBuilder
      • FieldBuilder
      • Emit
      • Utilities
    • Avalanche.FileSystem
      • Introduction
      • Abstractions
        • IFileSystem
          • IFileSystemBrowse
          • IFileSystemCreateDirectory
          • IFileSystemDelete
          • IFileSystemFileAttribute
          • IFileSystemMount
          • IFileSystemMove
          • IFileSystemObserve
          • IFileSystemOpen
        • IEvent
        • IEntry
        • IOption
        • IToken
      • FileSystem
      • VirtualFileSystem
      • MemoryFileSystem
      • EmbeddedFileSystem
      • HttpFileSystem
      • Decoration
      • IFileProvider
      • Events
      • Utilities
        • Dispose
        • File Scanner
        • Visit Tree
        • File Operation
    • Avalanche.Identity
      • Introduction
      • Identity
      • IdentityParts
      • IdentityInterner
      • IdentityComparer
      • Print Tree
      • IdentityAccessors
        • Introduction
        • TypeName
    • Avalanche.Localization
      • Introduction
      • Localization
      • LocalizationFile
      • LocalizationFiles
      • LocalizationFileSystem
      • LocalizationFileFormat
      • LocalizationLine
      • LocalizationLines
      • TemplateFormat
      • CultureProvider
      • FallbackCultureProvider
      • ResourceManager
      • LocalizationError
      • Microsoft.Extensions
        • Introduction
        • DependencyInjection
        • FileProvider
        • Logging
        • ITextLocalizer
        • IFileLocalizer
        • Localization
      • Asp.Net
        • Introduction
        • Supplying localization
        • Inject to pages
        • Culture Assigned
        • Minimalistic Api
        • Diagnostics
      • Pluralization
        • Introduction
        • Multiple plural parameters
        • Custom PluralRules
        • Invariant Culture
        • Unit Prefix
        • IPluralRule
        • IPluralNumber
        • IPluralRules
        • CLDRs
        • Unicode.CLDR40
        • Unicode.CLDR41
        • Unicode.CLDR42
      • Articles
        • Alphabet localization
        • Benchmarks
        • Caching
        • Class Library
        • Demo
        • Diagnostics
        • Embedded resources
        • Emplacement
        • File localization
        • Text localization
        • Printing templates
    • Avalanche.Message
      • Introduction
      • IMessage
      • IMessageProvider
      • IMessageDescription
      • IMessageDescriptions
      • MessageLevel
      • Message printing
      • Messages and Exceptions
      • Microsoft.Extensions
        • DependencyInjection
      • Articles
        • Aggregate Messages
        • Localization
        • Logging
        • Validation
    • Avalanche.Service
      • Introduction
      • Service
        • Introduction
        • IService
        • IServiceDisposable
        • IServiceDecoration
        • IServiceCast
        • IServiceObservable
        • IServiceContainer
        • Construction
        • Query
        • CancellationToken
        • CachePolicy
        • Scope
      • Handler
        • Introduction
        • IHandler
        • IHandlerCast
        • IHandlerDecoration
        • IHandlerWithOrder
        • CancellationToken
        • Cyclicity
        • Delegates
        • Invokable
        • ExportAttribute
        • OrderAttribute
        • PrintTree
        • Recursion
      • Query
        • Introduction
        • IQuery
        • IQueryCast
        • IQueryDecoration
      • Entry
        • Introduction
        • IEntry
        • IEntryCast
        • IEntryDecoration
        • IEntryObservable
        • IEntryVisitable
        • EntryState
      • Request
        • Introduction
        • IRequest
        • IRequestFor
        • IRequestToBeCached
        • IRequestToBeDisposed
        • RequestAttribute
        • ContextParameterAttribute
        • Print Tree
      • Dependency Injection
        • Introduction
        • Asp.Net
        • ServiceRequest<T>
        • Decorating a service
        • Handler
        • CachePolicy
        • CancellationToken
        • QueryLogger
      • Examples
        • NodeCount
        • Expression
        • Mapper
      • Articles
        • Benchmarks
        • Error Handling
        • ServiceMessages
    • Avalanche.StatusCode
      • Introduction
      • HResult
        • Introduction
        • HResult.Facilities
        • BasicMessages
        • RpcMessages
        • DispatchMessages
        • ItfMessages
        • Win32Messages
        • ClrMessages
      • System
        • Introduction
        • AccessControlMessages
        • AggregateMessages
        • AppDomainMessages
        • ArgumentMessages
        • ArgumentNullMessages
        • ArgumentOutOfRangeMessages
        • ArithmeticMessages
        • ArrayMessages
        • AssemblyMessages
        • BadImageFormatMessages
        • CodeContractMessages
        • CodePageMessages
        • CollectionsMessages
        • CompilerServiceMessages
        • CryptographyMessages
        • CultureMessages
        • DiagnosticsMessages
        • EventSourceMessages
        • ExecutionEngineMessages
        • FormatMessages
        • HostProtectionMessages
        • IOMessages
        • IndexOutOfRangeMessages
        • InteropServiceMessages
        • InvalidCastMessages
        • InvalidOperationMessages
        • IsolatedStorageMessages
        • LazyMessages
        • MarshalerMessages
        • MemoryMessages
        • MiscellaneousMessages
        • NotImplementedMessages
        • NotSupportedMessages
        • ObjectDisposedMessages
        • OperationCanceledMessages
        • OverflowMessages
        • PlatformMessages
        • PolicyMessages
        • PrincipalMessages
        • ProgramMessages
        • ReferenceMessages
        • ReflectionMessages
        • RegionMessages
        • RemotingMessages
        • ResourcesMessages
        • SecurityMessages
        • SerializationMessages
        • StackMessages
        • TaskMessages
        • TextMessages
        • ThreadingMessages
        • TimeZoneMessages
        • TypeMessages
        • XmlMessages
      • HttpStatusCode
      • OpcUaStatusCode
    • Avalanche.Template
      • Introduction
      • TemplateFormats
      • ITemplatePrintable
      • ITemplateFormatPrintable
      • ITemplateText
      • ITemplateBreakdown
      • ITemplateFormat
      • ITemplateFormats
      • Extract Arguments
      • Emplacement
    • Avalanche.Tokenizer
      • Introduction
      • IToken
      • ITokenizer
      • Tokenizers
    • Avalanche.Utilities
      • Introduction
      • Collections
        • Tuples
        • StructList
        • ArrayList
        • BijectionMap
        • LocakableDictionary
        • LockableList
        • MapList
        • Pipe
        • RingQueue
        • EnumerableExtensions
        • TupleUtilities
        • ArrayUtilities
      • Comparers
        • IGraphComparer
        • IGraphComparable
        • AlphaNumericComparer
        • EnumerableComparer
        • EnumerableGraphComparer
        • ReferenceComparer
        • KeyValuePairComparer
        • DefaultComparerProvider
        • RecordComparer
      • Cloners
        • ICloner
        • IGraphCloner
        • IGraphCloneable
        • ListCloner
        • DictionaryCloner
        • FieldCloner
        • PassthroughCloner
        • RecordCloner
        • ClonerProvider
      • Dispose
        • IDisposeAttachable
        • IDisposeBelatable
      • Provider
        • Introduction
        • ProviderBase
        • Delegate
        • Pipe
        • Cache
        • ResultCapture
        • AsReadOnly
        • AsService
        • IProviderEvent
      • Record
        • IRecordDescription
        • IFieldDescription
        • IConstructorDescription
        • IConstructionDescription
        • IParameterDescription
        • IRecordProviders
        • RecordDelegates
          • RecordCreate
          • RecordClone
          • RecordCopy
          • IRecordDelegates
        • FieldDelegates
          • FieldRead
          • FieldWrite
          • RecreateWith
          • IFieldDelegates
      • String
        • IEscaper
        • UnicodeString
      • Miscellaneous
        • IIdGenerator
        • Permutation
        • IReadOnly
        • IUserDataContainer
        • Void
    • Avalanche.Writer
      • Introduction
      • ConstantWriter
      • Context
      • ConvertWriter
      • DefaultConstructor
      • DelegateWriter
      • PassthroughWriter
      • Referer
      • TypeCast
      • Writer
      • WriterPipe
      • WriterMessages

    IToken

    IToken is interface for token types.

    /// <summary>Token features</summary>
    public interface IToken
    {
        /// <summary>Memory source</summary>
        ReadOnlyMemory<char> Memory { get; set; }
        /// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
        IToken[] Children { get; set; }
        /// <summary>Accept visitor.</summary>
        bool Accept(ITokenVisitor visitor);
    }
    
    /// <summary>Token visitor base interface.</summary>
    public interface ITokenVisitor { }
    /// <summary>Token visitor for <typeparamref name="T"/>.</summary>
    public interface ITokenVisitor<T> : ITokenVisitor where T : IToken
    {
        /// <summary>Visit token</summary>
        void Visit(T token);
    }
    


    Token type indicates the role of a character segment.

    // The tokenized text
    ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
    // Create composite token manually, for purpose of example
    CompositeToken compositeToken = new CompositeToken
    {
        Memory = text,
        Children = new IToken[]
        {
            new DecimalToken { Memory = text.Slice(0,1) },
            new WhitespaceToken { Memory = text.Slice(1,1) },
            new OperandToken { Memory = text.Slice(2, 1) },
            new WhitespaceToken { Memory = text.Slice(3,1) },
            new DecimalToken { Memory = text.Slice(4,1) },
            new WhitespaceToken { Memory = text.Slice(5,1) },
            new OperandToken { Memory = text.Slice(6, 1) },
            new WhitespaceToken { Memory = text.Slice(7,1) },
            new DecimalToken { Memory = text.Slice(8,1) },
        }
    };
    

    .PrintTree() extension method prints structural token as a tree.

    WriteLine(compositeToken.PrintTree());
    
    CompositeToken:  "1 + 2 = 3"
    ├── DecimalToken:  "1"
    ├── WhitespaceToken:  " "
    ├── OperandToken:  "+"
    ├── WhitespaceToken:  " "
    ├── DecimalToken:  "2"
    ├── WhitespaceToken:  " "
    ├── OperandToken:  "="
    ├── WhitespaceToken:  " "
    └── DecimalToken:  "3"
    

    .PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong) prints with indices.

    WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
    
    [0:9] CompositeToken:  "1 + 2 = 3"
    ├── [0:1] DecimalToken:  "1"
    ├── [1:2] WhitespaceToken:  " "
    ├── [2:3] OperandToken:  "+"
    ├── [3:4] WhitespaceToken:  " "
    ├── [4:5] DecimalToken:  "2"
    ├── [5:6] WhitespaceToken:  " "
    ├── [6:7] OperandToken:  "="
    ├── [7:8] WhitespaceToken:  " "
    └── [8:9] DecimalToken:  "3"
    

    .VisitTree() visits each tree node.

    foreach (var line in compositeToken.VisitTree())
        WriteLine(line);
    

    .Text() gets original string or creates a new string if represents a slice.

    string text_ = compositeToken.Text();
    

    .Memory.Index() and .Memory.Length return the spanned memory range.

    int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;
    

    .Accept(visitor) takes a visitor.

    compositeToken.Accept(new TokenVisitor());
    
    public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
    {
        public void Visit(CompositeToken token)
        {
            foreach (IToken child in token.Children)
                child.Accept(this);
        }
        public void Visit(DecimalToken token) => Console.WriteLine(token);
        public void Visit(OperandToken token) => Console.WriteLine(token);
    }
    
    [0:1] DecimalToken "1"
    [2:3] OperandToken "+"
    [4:5] DecimalToken "2"
    [6:7] OperandToken "="
    [8:9] DecimalToken "3"
    

    Implementation

    Implementation holds a ReadOnlyMemory<char> and optinally a child array.

    using Avalanche.Tokenizer;
    using Avalanche.Utilities;
    
    /// <summary>Example token.</summary>
    public struct IdentifierToken : IToken
    {
        /// <summary>Create token</summary>
        public IdentifierToken() { }
        /// <summary>Text source</summary>
        public ReadOnlyMemory<char> Memory { get; set; } = default;
        /// <summary>Children of structural token. Each child must be contained in the range of this parent token.</summary>
        public IToken[] Children { get; set; } = Array.Empty<IToken>();
        /// <summary>Accept visitor.</summary>
        public bool Accept(ITokenVisitor visitor) { if (visitor is ITokenVisitor<IdentifierToken> c) { c.Visit(this); return true; } else return false; }
        /// <summary>Print token as string</summary>
        public override string ToString() { int index = Memory.Index(); return $"[{index}:{index + Memory.Length}] {GetType().Name} \"{Memory}\""; }
    }
    
    

    The following tokens are introduced in Avalanche.Tokenizer.

    Token Description
    CompositeToken Structural token
    DecimalToken Decimal number
    KeyToken Key part
    KeyValueToken Key-value
    ValueToken Value part
    MalformedToken Malformed formatting
    NameToken Name
    NewLineToken Indicates '\n'
    OperandToken Operand, such as '%', '+', etc
    ParenthesisToken '(' or ')'
    RangeToken Range, e.g. "0..2"
    SeparatorToken Separator, e.g. ','
    TextToken Text part
    VariableToken Variable name
    WhitespaceToken Whitespace part
    GroupToken Group
    GroupSeparatorToken Group separator

    Full Example

    Full example
    using System.Globalization;
    using Avalanche.Tokenizer;
    using Avalanche.Utilities;
    using static System.Console;
    
    public class token
    {
        public static void Run()
        {
            {
                // <01>
                // The tokenized text
                ReadOnlyMemory<char> text = "1 + 2 = 3".AsMemory();
                // Create composite token manually, for purpose of example
                CompositeToken compositeToken = new CompositeToken
                {
                    Memory = text,
                    Children = new IToken[]
                    {
                        new DecimalToken { Memory = text.Slice(0,1) },
                        new WhitespaceToken { Memory = text.Slice(1,1) },
                        new OperandToken { Memory = text.Slice(2, 1) },
                        new WhitespaceToken { Memory = text.Slice(3,1) },
                        new DecimalToken { Memory = text.Slice(4,1) },
                        new WhitespaceToken { Memory = text.Slice(5,1) },
                        new OperandToken { Memory = text.Slice(6, 1) },
                        new WhitespaceToken { Memory = text.Slice(7,1) },
                        new DecimalToken { Memory = text.Slice(8,1) },
                    }
                };
                // </01>
                // <02>
                WriteLine(compositeToken.PrintTree());
                // </02>
                // <02B>
                WriteLine(compositeToken.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
                // </02B>
                // <03>
                foreach (var line in compositeToken.VisitTree())
                    WriteLine(line);
                // </03>
                // <04>
                string text_ = compositeToken.Text();
                // </04>
                // <05>
                int index = compositeToken.Memory.Index(), length = compositeToken.Memory.Length;
                // </05>
                // <06>
                compositeToken.Accept(new TokenVisitor());
                // </06>
            }
        }
    
        // <99>
        public class TokenVisitor : ITokenVisitor<CompositeToken>, ITokenVisitor<DecimalToken>, ITokenVisitor<OperandToken>
        {
            public void Visit(CompositeToken token)
            {
                foreach (IToken child in token.Children)
                    child.Accept(this);
            }
            public void Visit(DecimalToken token) => Console.WriteLine(token);
            public void Visit(OperandToken token) => Console.WriteLine(token);
        }
        // </99>
    }
    
    
    In This Article
    Back to top Copyright © Toni Kalajainen