1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

require 'rubygems'
require 'test/unit'
require File.dirname(__FILE__) + '/../init'
require 'active_support'

class MultilingualFerretTools::AnalyzerTest < Test::Unit::TestCase
  def test_all_latin
    a = MultilingualFerretTools::Analyzer.new
    ts = a.token_stream 'foo', 'now is the time'
    
    assert_next_token ts, 'now', 0, 3
    assert_next_token ts, 'time', 11, 15
  end
  
  def test_all_non_latin
    a = MultilingualFerretTools::Analyzer.new
    ts = a.token_stream 'foo', "\xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86"
    
    assert_next_token ts, "\xe6\x82\xaa", 0, 3
    assert_next_token ts, "\xe3\x81\x9d", 3, 6
    assert_next_token ts, "\xe3\x81\x86", 6, 9
  end
  
  def test_mixed
    a = MultilingualFerretTools::Analyzer.new
    ts = a.token_stream 'foo', "\xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86 foo and bar \xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86"

    assert_next_token ts, "\xe6\x82\xaa", 0, 3
    assert_next_token ts, "\xe3\x81\x9d", 3, 6
    assert_next_token ts, "\xe3\x81\x86", 6, 9
    assert_next_token ts, 'foo', 10, 13
    assert_next_token ts, 'bar', 18, 21
    assert_next_token ts, "\xe6\x82\xaa", 22, 25
    assert_next_token ts, "\xe3\x81\x9d", 25, 28
    assert_next_token ts, "\xe3\x81\x86", 28, 31
  end
  
  private
  
  def assert_next_token(ts, text, start, _end)
    token = ts.next
    assert_equal token.text, text
    assert_equal token.start, start
    assert_equal token.end, _end
  end
end