Project · Current .NET

Reusable separator tokenizer lab

Use one SearchValues<char> set to split owned nonempty tokens and verify six concrete representation and boundary cases.

Open project in playground ← All projects

All C# source files below are loaded together in the playground. Files written by a run live in its virtual directory.

Program.cs

var checks = TokenizerChecks.Run();
foreach (var check in checks) Console.WriteLine($"{check.Name}:{(check.Passed ? "PASS" : "FAIL")}");
Console.WriteLine($"passed:{checks.Count(check => check.Passed)}/{checks.Count}");
return checks.All(check => check.Passed) ? 0 : 1;

Domain/Tokenizer.cs

using System.Buffers;
public static class Tokenizer
{
    private static readonly SearchValues<char> Separators = SearchValues.Create(",;|");
    public static string[] SplitNonEmpty(string text)
    {
        ArgumentNullException.ThrowIfNull(text);
        ReadOnlySpan<char> remaining = text.AsSpan();
        List<string> tokens = [];
        while (!remaining.IsEmpty)
        {
            int index = remaining.IndexOfAny(Separators);
            ReadOnlySpan<char> token = index < 0 ? remaining : remaining[..index];
            token = token.Trim();
            if (!token.IsEmpty) tokens.Add(token.ToString());
            if (index < 0) break;
            remaining = remaining[(index + 1)..];
        }
        return [.. tokens];
    }
}

Checks/TokenizerChecks.cs

public static class TokenizerChecks
{
    public static List<(string Name, bool Passed)> Run()
    {
        var cases = new (string Name, string Input, string[] Expected)[]
        {
            ("mixed", "a,b;c|d", ["a", "b", "c", "d"]),
            ("empty", "", []),
            ("adjacent", "a,,b", ["a", "b"]),
            ("edge", ";a|", ["a"]),
            ("spaces", " a ;  b  ", ["a", "b"]),
            ("unicode", "rune;🦀", ["rune", "🦀"])
        };
        List<(string Name, bool Passed)> checks = [];
        foreach (var item in cases)
            checks.Add((item.Name, Tokenizer.SplitNonEmpty(item.Input).SequenceEqual(item.Expected)));
        return checks;
    }
}

ConsoleLab.csproj

<Project Sdk="Microsoft.NET.Sdk">
  <PropertyGroup>
    <OutputType>Exe</OutputType>
    <TargetFramework>net9.0</TargetFramework>
    <LangVersion>13.0</LangVersion>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
</Project>