• Avalanche.Core
Search Results for

    Show / Hide Table of Contents
    • Avalanche.Accessor
      • Introduction
      • IAccessor
        • IAccessor
        • IListAccessor
        • IMapAccessor
        • IRecordAccessor
        • IContentAccessor
        • IOneOfAccessor
        • IAnyAccessor
      • .Net
        • Introduction
        • IList<T>
        • IDictionary<K,V>
        • FieldInfo
        • OneOfAttribute
        • StructLayoutAttribute
        • Class
      • Protobuf
        • Introduction
      • Articles
        • Dependency Injection
        • AccessorMessages
    • Avalanche.Binding
      • Introduction
    • Avalanche.Core
      • License
    • Avalanche.DataType
      • Introduction
      • DataType
        • IDataType
        • IListType
        • IMapType
        • IRecordType
        • IFieldType
        • IOneOfType
        • IAnyType
        • IStringType
        • IValueType
        • IIntegerType
        • IEnumerationType
        • IRealType
      • .Net
        • Introduction
        • IList<T>
        • IDictionary<K,V>
        • FieldInfo
        • Enum
        • OneOfAttribute
        • StructLayoutAttribute
        • Class
      • Protobuf
        • Introduction
      • Articles
        • DataTypeRequest
        • PrintTree
        • DataTypeMessages
    • Avalanche.Emit
      • Introduction
      • TypeBuilder
      • ConstructorBuilder
      • MethodBuilder
      • PropertyBuilder
      • FieldBuilder
      • Emit
      • Utilities
    • Avalanche.FileSystem
      • Introduction
      • Abstractions
        • IFileSystem
          • IFileSystemBrowse
          • IFileSystemCreateDirectory
          • IFileSystemDelete
          • IFileSystemFileAttribute
          • IFileSystemMount
          • IFileSystemMove
          • IFileSystemObserve
          • IFileSystemOpen
        • IEvent
        • IEntry
        • IOption
        • IToken
      • FileSystem
      • VirtualFileSystem
      • MemoryFileSystem
      • EmbeddedFileSystem
      • HttpFileSystem
      • Decoration
      • IFileProvider
      • Events
      • Utilities
        • Dispose
        • File Scanner
        • Visit Tree
        • File Operation
    • Avalanche.Identity
      • Introduction
      • Identity
      • IdentityParts
      • IdentityInterner
      • IdentityComparer
      • Print Tree
      • IdentityAccessors
        • Introduction
        • TypeName
    • Avalanche.Localization
      • Introduction
      • Localization
      • LocalizationFile
      • LocalizationFiles
      • LocalizationFileSystem
      • LocalizationFileFormat
      • LocalizationLine
      • LocalizationLines
      • TemplateFormat
      • CultureProvider
      • FallbackCultureProvider
      • ResourceManager
      • LocalizationError
      • Microsoft.Extensions
        • Introduction
        • DependencyInjection
        • FileProvider
        • Logging
        • ITextLocalizer
        • IFileLocalizer
        • Localization
      • Asp.Net
        • Introduction
        • Supplying localization
        • Inject to pages
        • Culture Assigned
        • Minimalistic Api
        • Diagnostics
      • Pluralization
        • Introduction
        • Multiple plural parameters
        • Custom PluralRules
        • Invariant Culture
        • Unit Prefix
        • IPluralRule
        • IPluralNumber
        • IPluralRules
        • CLDRs
        • Unicode.CLDR40
        • Unicode.CLDR41
        • Unicode.CLDR42
      • Articles
        • Alphabet localization
        • Benchmarks
        • Caching
        • Class Library
        • Demo
        • Diagnostics
        • Embedded resources
        • Emplacement
        • File localization
        • Text localization
        • Printing templates
    • Avalanche.Message
      • Introduction
      • IMessage
      • IMessageProvider
      • IMessageDescription
      • IMessageDescriptions
      • MessageLevel
      • Message printing
      • Messages and Exceptions
      • Microsoft.Extensions
        • DependencyInjection
      • Articles
        • Aggregate Messages
        • Localization
        • Logging
        • Validation
    • Avalanche.Service
      • Introduction
      • Service
        • Introduction
        • IService
        • IServiceDisposable
        • IServiceDecoration
        • IServiceCast
        • IServiceObservable
        • IServiceContainer
        • Construction
        • Query
        • CancellationToken
        • CachePolicy
        • Scope
      • Handler
        • Introduction
        • IHandler
        • IHandlerCast
        • IHandlerDecoration
        • IHandlerWithOrder
        • CancellationToken
        • Cyclicity
        • Delegates
        • Invokable
        • ExportAttribute
        • OrderAttribute
        • PrintTree
        • Recursion
      • Query
        • Introduction
        • IQuery
        • IQueryCast
        • IQueryDecoration
      • Entry
        • Introduction
        • IEntry
        • IEntryCast
        • IEntryDecoration
        • IEntryObservable
        • IEntryVisitable
        • EntryState
      • Request
        • Introduction
        • IRequest
        • IRequestFor
        • IRequestToBeCached
        • IRequestToBeDisposed
        • RequestAttribute
        • ContextParameterAttribute
        • Print Tree
      • Dependency Injection
        • Introduction
        • Asp.Net
        • ServiceRequest<T>
        • Decorating a service
        • Handler
        • CachePolicy
        • CancellationToken
        • QueryLogger
      • Examples
        • NodeCount
        • Expression
        • Mapper
      • Articles
        • Benchmarks
        • Error Handling
        • ServiceMessages
    • Avalanche.StatusCode
      • Introduction
      • HResult
        • Introduction
        • HResult.Facilities
        • BasicMessages
        • RpcMessages
        • DispatchMessages
        • ItfMessages
        • Win32Messages
        • ClrMessages
      • System
        • Introduction
        • AccessControlMessages
        • AggregateMessages
        • AppDomainMessages
        • ArgumentMessages
        • ArgumentNullMessages
        • ArgumentOutOfRangeMessages
        • ArithmeticMessages
        • ArrayMessages
        • AssemblyMessages
        • BadImageFormatMessages
        • CodeContractMessages
        • CodePageMessages
        • CollectionsMessages
        • CompilerServiceMessages
        • CryptographyMessages
        • CultureMessages
        • DiagnosticsMessages
        • EventSourceMessages
        • ExecutionEngineMessages
        • FormatMessages
        • HostProtectionMessages
        • IOMessages
        • IndexOutOfRangeMessages
        • InteropServiceMessages
        • InvalidCastMessages
        • InvalidOperationMessages
        • IsolatedStorageMessages
        • LazyMessages
        • MarshalerMessages
        • MemoryMessages
        • MiscellaneousMessages
        • NotImplementedMessages
        • NotSupportedMessages
        • ObjectDisposedMessages
        • OperationCanceledMessages
        • OverflowMessages
        • PlatformMessages
        • PolicyMessages
        • PrincipalMessages
        • ProgramMessages
        • ReferenceMessages
        • ReflectionMessages
        • RegionMessages
        • RemotingMessages
        • ResourcesMessages
        • SecurityMessages
        • SerializationMessages
        • StackMessages
        • TaskMessages
        • TextMessages
        • ThreadingMessages
        • TimeZoneMessages
        • TypeMessages
        • XmlMessages
      • HttpStatusCode
      • OpcUaStatusCode
    • Avalanche.Template
      • Introduction
      • TemplateFormats
      • ITemplatePrintable
      • ITemplateFormatPrintable
      • ITemplateText
      • ITemplateBreakdown
      • ITemplateFormat
      • ITemplateFormats
      • Extract Arguments
      • Emplacement
    • Avalanche.Tokenizer
      • Introduction
      • IToken
      • ITokenizer
      • Tokenizers
    • Avalanche.Utilities
      • Introduction
      • Collections
        • Tuples
        • StructList
        • ArrayList
        • BijectionMap
        • LocakableDictionary
        • LockableList
        • MapList
        • Pipe
        • RingQueue
        • EnumerableExtensions
        • TupleUtilities
        • ArrayUtilities
      • Comparers
        • IGraphComparer
        • IGraphComparable
        • AlphaNumericComparer
        • EnumerableComparer
        • EnumerableGraphComparer
        • ReferenceComparer
        • KeyValuePairComparer
        • DefaultComparerProvider
        • RecordComparer
      • Cloners
        • ICloner
        • IGraphCloner
        • IGraphCloneable
        • ListCloner
        • DictionaryCloner
        • FieldCloner
        • PassthroughCloner
        • RecordCloner
        • ClonerProvider
      • Dispose
        • IDisposeAttachable
        • IDisposeBelatable
      • Provider
        • Introduction
        • ProviderBase
        • Delegate
        • Pipe
        • Cache
        • ResultCapture
        • AsReadOnly
        • AsService
        • IProviderEvent
      • Record
        • IRecordDescription
        • IFieldDescription
        • IConstructorDescription
        • IConstructionDescription
        • IParameterDescription
        • IRecordProviders
        • RecordDelegates
          • RecordCreate
          • RecordClone
          • RecordCopy
          • IRecordDelegates
        • FieldDelegates
          • FieldRead
          • FieldWrite
          • RecreateWith
          • IFieldDelegates
      • String
        • IEscaper
        • UnicodeString
      • Miscellaneous
        • IIdGenerator
        • Permutation
        • IReadOnly
        • IUserDataContainer
        • Void
    • Avalanche.Writer
      • Introduction
      • ConstantWriter
      • Context
      • ConvertWriter
      • DefaultConstructor
      • DelegateWriter
      • PassthroughWriter
      • Referer
      • TypeCast
      • Writer
      • WriterPipe
      • WriterMessages

    Tokenizers

    AllTokenizer takes all remaining characters.

    var tokenizer = AllTokenizer<TextToken>.Instance;
    TextToken token = tokenizer.Take<TextToken>("123456");
    // [0:6] TextToken:  "123456"
    

    AnyTokenizer(tokenizers) takes with any of the sub-tokenizers.

    var tokenizer = new AnyTokenizer<TextToken>(IntegerTokenizer.Instance, WhitespaceTokenizer.Any);
    TextToken token = tokenizer.Take<TextToken>("1234567890");
    // [0:10] TextToken:  "1234567890"
    

    CharTokenizer(Func<Memory,bool>) uses delegate to evaluate which characters to take.

    var tokenizer = new CharTokenizer<TextToken>(mem => mem.Span[0] == '_');
    TextToken token = tokenizer.Take<TextToken>("____abc");
    // [0:4] TextToken:  "____"
    

    ConstantTokenizer(string) takes a specific string.

    var tokenizer = new ConstantTokenizer<SeparatorToken>(",");
    SeparatorToken token = tokenizer.Take<SeparatorToken>(",x");
    // [0:1] SeparatorToken:  ","
    

    FuncTokenizer(Func<Memory, Token>) adapts func into tokenizer.

    var tokenizer = new FuncTokenizer<TextToken>(mem => new TextToken { Memory = mem.Slice(0, 3) });
    TextToken token = tokenizer.Take<TextToken>("____abc");
    // [0:3] TextToken:  "___"
    

    Tokenizers.Func(Func<Memory, Token>) adapts func into tokenizer.

    var tokenizer = Tokenizers.Func<TextToken>(mem => new TextToken { Memory = mem.Slice(0, 3) });
    TextToken token = tokenizer.Take<TextToken>("____abc");
    // [0:3] TextToken:  "___"
    

    HexTokenizer.WithoutPrefix tokenizes hexadecimal string.

    var tokenizer = HexTokenizer<ValueToken>.WithoutPrefix;
    IToken token = tokenizer.Take<IToken>("0123456789ABCDEF")!;
    // [0:16] ValueToken:  "0123456789ABCDEF"
    

    HexTokenizer.WithPrefix tokenizes hexadecimal string with '0x' prefix.

    var tokenizer = HexTokenizer<DecimalToken>.WithPrefix;
    IToken token = tokenizer.Take<IToken>("0x0123456789ABCDEF")!;
    // [0:18] DecimalToken:  "0x0123456789ABCDEF"
    

    IntegerTokenizer.Instance tokenizes integer decimals characters.

    var tokenizer = IntegerTokenizer<ValueToken>.Instance;
    IToken token = tokenizer.Take<IToken>("0123456789")!;
    // [0:10] ValueToken:  "0123456789"
    

    MalformedTokenizer.Instance takes one following character. Typically used as last case in AnyTokenizer.

    var tokenizer = MalformedTokenizer<MalformedToken>.Instance;
    IToken token = tokenizer.Take<IToken>("¤§")!;
    // [0:1] MalformedToken:  "¤"
    

    NewLineTokenizer.Instance tokenizes new line character '\n'.

    var tokenizer = NewLineTokenizer<NewLineToken>.Instance;
    IToken token = tokenizer.Take<IToken>("\n\n\n")!;
    // [0:1] NewLineToken:  "\n"
    

    RealTokenizer.Instance tokenies real number.

    var tokenizer = RealTokenizer<DecimalToken>.Instance;
    IToken token = tokenizer.Take<IToken>("-123.45600e-12 asdf")!;
    // [0:14] DecimalToken:  "-123.45600e-12"
    

    RegexTokenizer tokenizes using regex pattern. Note that pattern typically starts with "^" to indicate beginning characters.

    var tokenizer = new RegexTokenizer<TextToken>("^[a-zA-Z0-9]+");
    IToken token = tokenizer.Take<IToken>("ab12 cd34")!;
    // [0:4] TextToken:  "ab12"
    

    SequenceTokenizer(tokenizers) tokenizes a specific sequence of sub-tokenizers.

    var tokenizer = new SequenceTokenizer<CompositeToken>(
            IntegerTokenizer.Instance,
            new ConstantTokenizer<SeparatorToken>("="),
            IntegerTokenizer.Instance
        );
    IToken token = tokenizer.Take<IToken>("10=20")!;
    //[0:5] CompositeToken:  "10=20"
    //├── [0:2] DecimalToken: "10"
    //├── [2:3] SeparatorToken: "="
    //└── [3:5] DecimalToken: "20"
    

    SequenceTokenizer( (tokenizer, bool required)[] ) tokenizes a specific sequence of sub-tokenizers. Sub-tokenizer is optional.

    var tokenizer = new SequenceTokenizer<CompositeToken>(
            (WhitespaceTokenizer.Any, false),
            (IntegerTokenizer.Instance, true),
            (WhitespaceTokenizer.Any, false),
            (new ConstantTokenizer<SeparatorToken>("="), true),
            (WhitespaceTokenizer.Any, false),
            (IntegerTokenizer.Instance, true),
            (WhitespaceTokenizer.Any, false)
        );
    IToken token = tokenizer.Take<IToken>("10=20")!;
    //[0:5] CompositeToken:  "10=20"
    //├── [0:2] DecimalToken: "10"
    //├── [2:3] SeparatorToken: "="
    //└── [3:5] DecimalToken: "20"
    

    SequenceTokenizer( (tokenizer, bool required, bool yieldChildren)[] ) tokenizes a specific sequence of sub-tokenizers. Sub-tokenizer is optional. Can yield children of child tokens.

    var commaTokenizer = new ConstantTokenizer<OperandToken>(",");
    var anyTokenizer = new AnyTokenizer(commaTokenizer, new UntilTokenizer(commaTokenizer));
    var tokenizer = 
    new SequenceTokenizer<CompositeToken>(
            (WhitespaceTokenizer.Any, false, false),
            (anyTokenizer, true, true),
            (WhitespaceTokenizer.Any, false, false),
            (anyTokenizer, true, true),
            (WhitespaceTokenizer.Any, false, false)
        );
    IToken token = tokenizer.Take<IToken>("A,B")!;
    //[0:2] CompositeToken: "A,"
    //├── [0:1] TextToken: "A"
    //└── [1:2] OperandToken: ","
    

    TrueTokenizer decorates tokenizer so that it always returns true, even to empty token.

    var tokenizer = new TrueTokenizer<TextToken>(new RegexTokenizer<TextToken>("^[a-zA-Z0-9]?"));
    TextToken token = tokenizer.Take<TextToken>("");
    // [0:0] TextToken:  ""
    

    UntilTokenizer(tokenizer) tokenizes characters until end condition is met. End condition is another tokenizer. End condition is excluded from resulted token.

    var tokenizer = new UntilTokenizer<CompositeToken>(NewLineTokenizer.Instance);
    IToken token = tokenizer.Take<IToken>("First line\nSecond line\nThird line")!;
    // [0:10] CompositeToken:  "First line"
    

    UntilTokenizer(tokenizer, endOfSpan, char escapeChar) tokenizes characters until end condition is met. Until tokenize can be used as escape aware.

    var tokenizer = new UntilTokenizer(new ConstantTokenizer(" "), false, '\\');
    IToken token = tokenizer.Take<IToken>(@"a\ b\ c d e f")!;
    // [0:7] TextToken:  "a\\ b\\ c"
    

    WhileTokenizer(tokenizer, bool yieldChildren) tokenizes while its sub-tokenizer yields tokens.

    var tokenizer = new WhileTokenizer<CompositeToken>(
            new AnyTokenizer(
                new RegexTokenizer<TextToken>("^[a-zA-Z0-9]+"),
                WhitespaceTokenizer.AllButNewLine,
                NewLineTokenizer.Instance
            ),
            yieldChildren: true
        );
    CompositeToken token = tokenizer.TakeAll<CompositeToken>("Hello world\nabc\n123");
    // [0:19] CompositeToken: "Hello world\nabc\n123"
    // ├── [0:5] TextToken: "Hello"
    // ├── [5:6] WhitespaceToken: " "
    // ├── [6:11] TextToken: "world"
    // ├── [11:12] NewLineToken: "\n"
    // ├── [12:15] TextToken: "abc"
    // ├── [15:16] NewLineToken: "\n"
    // └── [16:19] TextToken: "123"
    

    WhitespaceTokenizer.Any tokenizes white-space characters.

    var tokenizer = WhitespaceTokenizer.Any;
    IToken token = tokenizer.Take<WhitespaceToken>("  \t\t  \n  ABC");
    // [0:9] WhitespaceToken:  "                 \n  "
    

    WhitespaceTokenizer.AllButNewLine tokenizes white-space characters except new-line '\n'.

    var tokenizer = WhitespaceTokenizer.AllButNewLine;
    IToken token = tokenizer.Take<WhitespaceToken>("  \t\t  \n  ABC");
    // [0:6] WhitespaceToken:  "                 "
    

    Full Example

    Full example
    using Avalanche.Tokenizer;
    
    public class tokenizers
    {
        public static void Run()
        {
            {
                // <01>
                var tokenizer = AllTokenizer<TextToken>.Instance;
                TextToken token = tokenizer.Take<TextToken>("123456");
                // [0:6] TextToken:  "123456"
                // </01>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <02>
                var tokenizer = new AnyTokenizer<TextToken>(IntegerTokenizer.Instance, WhitespaceTokenizer.Any);
                TextToken token = tokenizer.Take<TextToken>("1234567890");
                // [0:10] TextToken:  "1234567890"
                // </02>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <03>
                var tokenizer = new CharTokenizer<TextToken>(mem => mem.Span[0] == '_');
                TextToken token = tokenizer.Take<TextToken>("____abc");
                // [0:4] TextToken:  "____"
                // </03>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <04>
                var tokenizer = new ConstantTokenizer<SeparatorToken>(",");
                SeparatorToken token = tokenizer.Take<SeparatorToken>(",x");
                // [0:1] SeparatorToken:  ","
                // </04>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <05>
                var tokenizer = new FuncTokenizer<TextToken>(mem => new TextToken { Memory = mem.Slice(0, 3) });
                TextToken token = tokenizer.Take<TextToken>("____abc");
                // [0:3] TextToken:  "___"
                // </05>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <05B>
                var tokenizer = Tokenizers.Func<TextToken>(mem => new TextToken { Memory = mem.Slice(0, 3) });
                TextToken token = tokenizer.Take<TextToken>("____abc");
                // [0:3] TextToken:  "___"
                // </05B>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <06A>
                var tokenizer = HexTokenizer<ValueToken>.WithoutPrefix;
                IToken token = tokenizer.Take<IToken>("0123456789ABCDEF")!;
                // [0:16] ValueToken:  "0123456789ABCDEF"
                // </06A>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <06B>
                var tokenizer = HexTokenizer<DecimalToken>.WithPrefix;
                IToken token = tokenizer.Take<IToken>("0x0123456789ABCDEF")!;
                // [0:18] DecimalToken:  "0x0123456789ABCDEF"
                // </06B>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
    
            {
                // <07>
                var tokenizer = IntegerTokenizer<ValueToken>.Instance;
                IToken token = tokenizer.Take<IToken>("0123456789")!;
                // [0:10] ValueToken:  "0123456789"
                // </07>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <08>
                var tokenizer = MalformedTokenizer<MalformedToken>.Instance;
                IToken token = tokenizer.Take<IToken>("¤§")!;
                // [0:1] MalformedToken:  "¤"
                // </08>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <09>
                var tokenizer = NewLineTokenizer<NewLineToken>.Instance;
                IToken token = tokenizer.Take<IToken>("\n\n\n")!;
                // [0:1] NewLineToken:  "\n"
                // </09>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <10>
                var tokenizer = RealTokenizer<DecimalToken>.Instance;
                IToken token = tokenizer.Take<IToken>("-123.45600e-12 asdf")!;
                // [0:14] DecimalToken:  "-123.45600e-12"
                // </10>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <11>
                var tokenizer = new RegexTokenizer<TextToken>("^[a-zA-Z0-9]+");
                IToken token = tokenizer.Take<IToken>("ab12 cd34")!;
                // [0:4] TextToken:  "ab12"
                // </11>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <12A>
                var tokenizer = new SequenceTokenizer<CompositeToken>(
                        IntegerTokenizer.Instance,
                        new ConstantTokenizer<SeparatorToken>("="),
                        IntegerTokenizer.Instance
                    );
                IToken token = tokenizer.Take<IToken>("10=20")!;
                //[0:5] CompositeToken:  "10=20"
                //├── [0:2] DecimalToken: "10"
                //├── [2:3] SeparatorToken: "="
                //└── [3:5] DecimalToken: "20"
                // </12A>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <12B>
                var tokenizer = new SequenceTokenizer<CompositeToken>(
                        (WhitespaceTokenizer.Any, false),
                        (IntegerTokenizer.Instance, true),
                        (WhitespaceTokenizer.Any, false),
                        (new ConstantTokenizer<SeparatorToken>("="), true),
                        (WhitespaceTokenizer.Any, false),
                        (IntegerTokenizer.Instance, true),
                        (WhitespaceTokenizer.Any, false)
                    );
                IToken token = tokenizer.Take<IToken>("10=20")!;
                //[0:5] CompositeToken:  "10=20"
                //├── [0:2] DecimalToken: "10"
                //├── [2:3] SeparatorToken: "="
                //└── [3:5] DecimalToken: "20"
                // </12B>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <12C>
                var commaTokenizer = new ConstantTokenizer<OperandToken>(",");
                var anyTokenizer = new AnyTokenizer(commaTokenizer, new UntilTokenizer(commaTokenizer));
                var tokenizer = 
                new SequenceTokenizer<CompositeToken>(
                        (WhitespaceTokenizer.Any, false, false),
                        (anyTokenizer, true, true),
                        (WhitespaceTokenizer.Any, false, false),
                        (anyTokenizer, true, true),
                        (WhitespaceTokenizer.Any, false, false)
                    );
                IToken token = tokenizer.Take<IToken>("A,B")!;
                //[0:2] CompositeToken: "A,"
                //├── [0:1] TextToken: "A"
                //└── [1:2] OperandToken: ","
                // </12C>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <13>
                var tokenizer = new TrueTokenizer<TextToken>(new RegexTokenizer<TextToken>("^[a-zA-Z0-9]?"));
                TextToken token = tokenizer.Take<TextToken>("");
                // [0:0] TextToken:  ""
                // </13>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <14A>
                var tokenizer = new UntilTokenizer<CompositeToken>(NewLineTokenizer.Instance);
                IToken token = tokenizer.Take<IToken>("First line\nSecond line\nThird line")!;
                // [0:10] CompositeToken:  "First line"
                // </14A>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <14B>
                var tokenizer = new UntilTokenizer(new ConstantTokenizer(" "), false, '\\');
                IToken token = tokenizer.Take<IToken>(@"a\ b\ c d e f")!;
                // [0:7] TextToken:  "a\\ b\\ c"
                // </14B>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <15>
                var tokenizer = new WhileTokenizer<CompositeToken>(
                        new AnyTokenizer(
                            new RegexTokenizer<TextToken>("^[a-zA-Z0-9]+"),
                            WhitespaceTokenizer.AllButNewLine,
                            NewLineTokenizer.Instance
                        ),
                        yieldChildren: true
                    );
                CompositeToken token = tokenizer.TakeAll<CompositeToken>("Hello world\nabc\n123");
                // [0:19] CompositeToken: "Hello world\nabc\n123"
                // ├── [0:5] TextToken: "Hello"
                // ├── [5:6] WhitespaceToken: " "
                // ├── [6:11] TextToken: "world"
                // ├── [11:12] NewLineToken: "\n"
                // ├── [12:15] TextToken: "abc"
                // ├── [15:16] NewLineToken: "\n"
                // └── [16:19] TextToken: "123"
                // </15>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <16A>
                var tokenizer = WhitespaceTokenizer.Any;
                IToken token = tokenizer.Take<WhitespaceToken>("  \t\t  \n  ABC");
                // [0:9] WhitespaceToken:  "                 \n  "
                // </16A>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
            {
                // <16B>
                var tokenizer = WhitespaceTokenizer.AllButNewLine;
                IToken token = tokenizer.Take<WhitespaceToken>("  \t\t  \n  ABC");
                // [0:6] WhitespaceToken:  "                 "
                // </16B>
                Console.WriteLine(token.PrintTree(format: TokenPrintTreeExtensions.PrintFormat.DefaultLong));
            }
        }
    }
    
    
    In This Article
    Back to top Copyright © Toni Kalajainen