無差別に技術をついばむ鳥

※完成時の状態
Public Enum TokenCategory
    Text
    Number
    Symbol
    UnKnown
End Enum

Public Structure Token
    'トークンの種類
    Private m_category As TokenCategory
    Public Property Category() As TokenCategory
        Get
            Return Me.m_category
        End Get
        Set(ByVal value As TokenCategory)
            Me.m_category = value
        End Set
    End Property

    'トークンテキスト
    Private m_text As String
    Public ReadOnly Property Text() As String
        Get
            Return Me.m_text
        End Get
    End Property

    Public Sub New(ByVal value As String)
        Me.m_category = TokenCategory.UnKnown
        Me.m_text = value
    End Sub
End Structure

----------------------------------------------------------------------------
Imports System
Imports System.Text
Imports System.Collections.Generic

Public Class Lexer

    Public Function GetTokens(ByVal text As String) As Token()
        '文字列を単語に分ける
        Dim str As Char = text(0)
        Dim tmp As Queue(Of Token) = New Queue(Of Token)
        Dim word As StringBuilder = New StringBuilder(text.Length)
        Dim readIndex As Integer = 0
        For readIndex = 0 To text.Length
            str = text(readIndex)
            If str = " " Then
                If word.Length <> 0 Then
                    tmp.Enqueue(New Token(word.ToString()))
                    word = New StringBuilder(text.Length - readIndex)
                End If
                 '空白を無視する
                While str = " "
                    readIndex += 1
                    If readIndex = text.Length Then
                        Exit While
                    End If
                    str = text(readIndex)
                End While
            End If
            word.Append(str)
        Next

         '結果を返す
        Dim result As Token()
        Dim max As Integer = tmp.Count - 1
        ReDim result(max)
        For i As Integer = 0 To max
            result(i) = tmp.Dequeue()
        Next
        Return result
    End Function
End Class

Imports NUnit.Framework
Imports FoolishTranslator

 Public Class LexerTest
    'テスト対象の生成手段を一つにする
    Private Shared Function GenereteTestTarget() As Lexer
        Return New Lexer()
    End Function

    '簡単な英文を正しいトークンに分けられるかテスト
    < Test() > Public Sub SimpleGetTonkensTest()
        Dim target As Lexer = GenereteTestTarget()
        Dim data As String = "   Tokyo is the capital of Japan.   "
        Dim words As Token() = target.GetTokens(data)
        Assert.IsNotNull( _ 
            words, "何しているの！空じゃないのぉぉぉぉ by ドリィ")
        Assert.AreEqual( _ 
            "Tokyo", words(0).Text,  _ 
            "1つ目の単語が間違っているぅぅ by ドリィ")
        Assert.AreEqual( _ 
            "is", words(1).Text,  _ 
            "2つ目の単語が間違っているぅぅ by ドリィ")
        Assert.AreEqual( _ 
            "the", words(2).Text,  _ 
            "3つ目の単語が間違っているぅぅ by ドリィ")
        Assert.AreEqual( _ 
            "capital", words(3).Text,  _ 
            "4つ目の単語が間違っているぅぅ by ドリィ")
        Assert.AreEqual( _ 
            "of", words(4).Text,  _ 
            "5つ目の単語が間違っているぅぅ by ドリィ")
        Assert.AreEqual( _ 
            "Japan", words(5).Text, _ 
             "6つ目の単語が間違っているぅぅ。" &  _ 
            vbLf & "やはりドットを分割するの忘れたわね♪ by ドリィ")
    End Sub
End Class