Project: Lingr tools

Revision: 36

Author: dburkes

Date: 15 May 2007 14:22:49

• /plugins/multilingual_ferret_tools/Rakefile

  ......	@@ -0,0 +1,22 @@
  	1	require 'rake'
  	2	require 'rake/testtask'
  	3	require 'rake/rdoctask'
  	4
  	5	desc 'Default: run unit tests.'
  	6	task :default => :test
  	7
  	8	desc 'Test the multilngual_ferret_tools plugin.'
  	9	Rake::TestTask.new(:test) do |t|
  	10	t.libs << 'lib'
  	11	t.pattern = 'test/**/*_test.rb'
  	12	t.verbose = true
  	13	end
  	14
  	15	desc 'Generate documentation for the multilngual_ferret_tools plugin.'
  	16	Rake::RDocTask.new(:rdoc) do |rdoc|
  	17	rdoc.rdoc_dir = 'rdoc'
  	18	rdoc.title = 'MultilngualFerretTools'
  	19	rdoc.options << '--line-numbers' << '--inline-source'
  	20	rdoc.rdoc_files.include('README')
  	21	rdoc.rdoc_files.include('lib/**/*.rb')
  	22	end

• /plugins/multilingual_ferret_tools/lib/multilingual_analyzer.rb

  ......	@@ -0,0 +1,75 @@
  	1	require 'ferret'
  	2
  	3	module MultilingualFerretTools
  	4	class Analyzer < Ferret::Analysis::Analyzer
  	5	def initialize(lower = true)
  	6	super(lower)
  	7	@lower = lower
  	8	end
  	9
  	10	def token_stream(field, str)
  	11	TokenStream.new(field, str, @lower)
  	12	end
  	13	end
  	14
  	15	class TokenStream < Ferret::Analysis::TokenStream
  	16	def initialize(field, str, lower)
  	17	@field = field
  	18	@lower = lower
  	19	self.text = str
  	20	end
  	21
  	22	def text=(text)
  	23	@input = text
  	24	@index = 0
  	25	@substreams = []
  	26	@token_offset = 0
  	27	build_substreams
  	28	end
  	29
  	30	def next
  	31	return nil if @index >= @substreams.length
  	32
  	33	ret = @substreams[@index][:ts].next
  	34
  	35	if !ret
  	36	@token_offset += @substreams[@index][:len]
  	37	@index += 1
  	38	self.next
  	39	else
  	40	ret.start += @token_offset
  	41	ret.end += @token_offset
  	42	ret
  	43	end
  	44	end
  	45
  	46	private
  	47
  	48	def build_substreams
  	49	# if string is completely latin or non-latin, use a single substream
  	50	#
  	51	single_token_stream = token_stream_for(@input)
  	52	@substreams.push({ :ts => single_token_stream, :len => @input.length}) and return if single_token_stream
  	53
  	54	# otherwise, build a series of substreams for the text
  	55	#
  	56	chunker = Chunker.new @input, :whitespace => :combine
  	57	while chunk = chunker.next do
  	58	@substreams.push({ :ts => token_stream_for(chunk[0]), :len => chunk[0].length })
  	59	end
  	60	end
  	61
  	62	def token_stream_for(str)
  	63	case Chunker.classify(str)
  	64	when :latin, :latin_whitespace
  	65	analyzer = Ferret::Analysis::StandardAnalyzer.new(Ferret::Analysis::ENGLISH_STOP_WORDS, @lower)
  	66
  	67	when :non_latin, :non_latin_whitespace
  	68	analyzer = Ferret::Analysis::RegExpAnalyzer.new(/./, @lower)
  	69	end
  	70
  	71	analyzer ? analyzer.token_stream(@field, str) : nil
  	72	end
  	73	end
  	74
  	75	end
  0	76	\ No newline at end of file

• /plugins/multilingual_ferret_tools/lib/multilingual_version.rb

  ......	@@ -0,0 +1,3 @@
  	1	module MultilingualFerretTools
  	2	VERSION = "0.1"
  	3	end
  0	4	\ No newline at end of file

• /plugins/multilingual_ferret_tools/init.rb

  ......	@@ -0,0 +1,6 @@
  	1	$KCODE = 'u'
  	2	require 'jcode'
  	3
  	4	require File.dirname(__FILE__) + '/lib/multilingual_chunker'
  	5	require File.dirname(__FILE__) + '/lib/multilingual_analyzer'
  	6	require File.dirname(__FILE__) + '/lib/multilingual_version'
  0	7	\ No newline at end of file

• /plugins/multilingual_ferret_tools/test/multilingual_chunker_test.rb

  ......	@@ -0,0 +1,166 @@
  	1	require 'rubygems'
  	2	require 'test/unit'
  	3	require File.dirname(__FILE__) + '/../lib/multilingual_chunker'
  	4	require 'active_support'
  	5
  	6	class MultilingualFerretTools::ChunkerTest < Test::Unit::TestCase
  	7	def test_classify_all_latin
  	8	assert_equal MultilingualFerretTools::Chunker.classify(@@LATIN_STRING), :latin
  	9	end
  	10
  	11	def test_classify_all_non_latin
  	12	assert_equal MultilingualFerretTools::Chunker.classify(@@NON_LATIN_STRING), :non_latin
  	13	end
  	14
  	15	def test_classify_mixed
  	16	assert_equal MultilingualFerretTools::Chunker.classify(@@LATIN_STRING + @@LATIN_WHITESPACE + @@NON_LATIN_STRING), :mixed
  	17	assert_equal MultilingualFerretTools::Chunker.classify(@@LATIN_STRING + @@NON_LATIN_WHITESPACE + @@NON_LATIN_STRING), :mixed
  	18	end
  	19
  	20	def test_classify_latin_whitespace
  	21	assert_equal MultilingualFerretTools::Chunker.classify(@@LATIN_WHITESPACE), :latin_whitespace
  	22	end
  	23
  	24	def test_classify_non_latin_whitespace
  	25	assert_equal MultilingualFerretTools::Chunker.classify(@@NON_LATIN_WHITESPACE), :non_latin_whitespace
  	26	end
  	27
  	28	def test_chunk_all_latin_with_discarded_whitespace
  	29	c = MultilingualFerretTools::Chunker.new @@LATIN_STRING, :whitespace => :discard
  	30	assert_latin_string_words c
  	31	end
  	32
  	33	def test_chunk_all_latin_with_combined_whitespace
  	34	c = MultilingualFerretTools::Chunker.new @@LATIN_STRING, :whitespace => :combine
  	35	assert_latin_string c
  	36	end
  	37
  	38	def test_chunk_all_latin_with_reported_whitespace
  	39	c = MultilingualFerretTools::Chunker.new @@LATIN_STRING, :whitespace => :report
  	40	assert_latin_string_words c, 0, true
  	41	end
  	42
  	43	def test_chunk_all_non_latin_single_word
  	44	c = MultilingualFerretTools::Chunker.new @@NON_LATIN_STRING, :whitespace => :discard
  	45	assert_non_latin_string c
  	46	end
  	47
  	48	def test_chunk_all_non_latin_with_discarded_whitespace
  	49	c = MultilingualFerretTools::Chunker.new @@NON_LATIN_STRING + @@NON_LATIN_WHITESPACE + @@NON_LATIN_STRING, :whitespace => :discard
  	50	n = assert_non_latin_string c
  	51	assert_non_latin_string c, n + @@NON_LATIN_WHITESPACE.length
  	52	end
  	53
  	54	def test_chunk_all_non_latin_with_reported_whitespace
  	55	c = MultilingualFerretTools::Chunker.new @@NON_LATIN_STRING + @@NON_LATIN_WHITESPACE + @@NON_LATIN_STRING, :whitespace => :report
  	56	n = assert_non_latin_string c
  	57	n = assert_non_latin_whitespace c, n
  	58	assert_non_latin_string c, n
  	59	end
  	60
  	61	def test_chunk_mixed_with_discarded_whitespace
  	62	c = MultilingualFerretTools::Chunker.new @@MIXED_STRING, :whitespace => :discard
  	63	n = assert_latin_string_words c
  	64	n = assert_non_latin_string c, n + @@LATIN_WHITESPACE.length
  	65	assert_latin_string_words c, n + @@NON_LATIN_WHITESPACE.length
  	66	end
  	67
  	68	def test_chunk_mixed_with_combined_whitespace
  	69	c = MultilingualFerretTools::Chunker.new @@MIXED_STRING, :whitespace => :combine
  	70	n = assert_latin_string c, 0, true
  	71	n = assert_non_latin_string c, n, true
  	72	assert_latin_string c, n
  	73	end
  	74
  	75	def test_chunk_mixed_with_reported_whitespace
  	76	c = MultilingualFerretTools::Chunker.new @@MIXED_STRING, :whitespace => :report
  	77	n = assert_latin_string_words c, 0, true
  	78	n = assert_latin_whitespace c, n
  	79	n = assert_non_latin_string c, n
  	80	n = assert_non_latin_whitespace c, n
  	81	assert_latin_string_words c, n, true
  	82	end
  	83
  	84	private
  	85
  	86	def assert_latin_string_words(chunker, start=0, expect_whitespace_chunks=false)
  	87	accumulated_length = 0
  	88	end_index = 0
  	89
  	90	@@LATIN_WORDS.each_with_index do |w, i|
  	91	chunk = chunker.next
  	92
  	93	assert_equal chunk[0], w
  	94	assert_equal chunk[1], :latin
  	95	assert_equal chunk[2], start + accumulated_length
  	96	assert_equal chunk[3], start + accumulated_length + w.length - 1
  	97
  	98	accumulated_length += w.length
  	99	accumulated_length += @@LATIN_WHITESPACE.length if !expect_whitespace_chunks
  	100	end_index = chunk[3] + 1
  	101
  	102	if i < @@LATIN_WORDS.length - 1 and expect_whitespace_chunks
  	103	end_index = assert_latin_whitespace(chunker, chunk[3] + 1)
  	104	accumulated_length += @@LATIN_WHITESPACE.length
  	105	end
  	106	end
  	107
  	108	end_index
  	109	end
  	110
  	111	def assert_latin_string(chunker, start=0, trailing_whitespace=false)
  	112	expected = trailing_whitespace ? @@LATIN_STRING + @@LATIN_WHITESPACE : @@LATIN_STRING
  	113
  	114	chunk = chunker.next
  	115	assert_equal chunk[0], expected
  	116	assert_equal chunk[1], :latin
  	117	assert_equal chunk[2], start
  	118	assert_equal chunk[3], start + expected.length - 1
  	119	chunk[3] + 1
  	120	end
  	121
  	122	def assert_latin_whitespace(chunker, start)
  	123	chunk = chunker.next
  	124	assert_equal chunk[0], @@LATIN_WHITESPACE
  	125	assert_equal chunk[1], :latin_whitespace
  	126	assert_equal chunk[2], start
  	127	assert_equal chunk[3], start + @@LATIN_WHITESPACE.length - 1
  	128	chunk[3] + 1
  	129	end
  	130
  	131	def assert_non_latin_string(chunker, start=0, trailing_whitespace=false)
  	132	expected = trailing_whitespace ? @@NON_LATIN_STRING + @@NON_LATIN_WHITESPACE : @@NON_LATIN_STRING
  	133	chunk = chunker.next
  	134	assert_equal chunk[0], expected
  	135	assert_equal chunk[1], :non_latin
  	136	assert_equal chunk[2], start
  	137	assert_equal chunk[3], start + expected.length - 1
  	138	chunk[3] + 1
  	139	end
  	140
  	141	def assert_non_latin_whitespace(chunker, start)
  	142	chunk = chunker.next
  	143	assert_equal chunk[0], @@NON_LATIN_WHITESPACE
  	144	assert_equal chunk[1], :non_latin_whitespace
  	145	assert_equal chunk[2], start
  	146	assert_equal chunk[3], start + @@NON_LATIN_WHITESPACE.length - 1
  	147	chunk[3] + 1
  	148	end
  	149
  	150	def assert_next_chunk(chunker, str, type, start, _end)
  	151	chunk = chunker.next
  	152	assert_equal chunk[0], str
  	153	assert_equal chunk[1], type
  	154	assert_equal chunk[2], start
  	155	assert_equal chunk[3], _end
  	156	end
  	157
  	158	@@LATIN_WORDS = [ "foo", "bar", "bro", "baz"]
  	159	@@LATIN_WHITESPACE = " "
  	160	@@LATIN_STRING = @@LATIN_WORDS.join(@@LATIN_WHITESPACE)
  	161
  	162	@@NON_LATIN_STRING = "\xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86"
  	163	@@NON_LATIN_WHITESPACE = "\xe3\x80\x80"
  	164
  	165	@@MIXED_STRING = @@LATIN_STRING + @@LATIN_WHITESPACE + @@NON_LATIN_STRING + @@NON_LATIN_WHITESPACE + @@LATIN_STRING
  	166	end

• /plugins/multilingual_ferret_tools/test/multilingual_analyzer_test.rb

  ......	@@ -0,0 +1,47 @@
  	1	require 'rubygems'
  	2	require 'test/unit'
  	3	require File.dirname(__FILE__) + '/../init'
  	4	require 'active_support'
  	5
  	6	class MultilingualFerretTools::AnalyzerTest < Test::Unit::TestCase
  	7	def test_all_latin
  	8	a = MultilingualFerretTools::Analyzer.new
  	9	ts = a.token_stream 'foo', 'now is the time'
  	10
  	11	assert_next_token ts, 'now', 0, 3
  	12	assert_next_token ts, 'time', 11, 15
  	13	end
  	14
  	15	def test_all_non_latin
  	16	a = MultilingualFerretTools::Analyzer.new
  	17	ts = a.token_stream 'foo', "\xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86"
  	18
  	19	assert_next_token ts, "\xe6\x82\xaa", 0, 3
  	20	assert_next_token ts, "\xe3\x81\x9d", 3, 6
  	21	assert_next_token ts, "\xe3\x81\x86", 6, 9
  	22	end
  	23
  	24	def test_mixed
  	25	a = MultilingualFerretTools::Analyzer.new
  	26	ts = a.token_stream 'foo', "\xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86 foo and bar \xe6\x82\xaa\xe3\x81\x9d\xe3\x81\x86"
  	27
  	28	assert_next_token ts, "\xe6\x82\xaa", 0, 3
  	29	assert_next_token ts, "\xe3\x81\x9d", 3, 6
  	30	assert_next_token ts, "\xe3\x81\x86", 6, 9
  	31	assert_next_token ts, 'foo', 10, 13
  	32	assert_next_token ts, 'bar', 18, 21
  	33	assert_next_token ts, "\xe6\x82\xaa", 22, 25
  	34	assert_next_token ts, "\xe3\x81\x9d", 25, 28
  	35	assert_next_token ts, "\xe3\x81\x86", 28, 31
  	36	end
  	37
  	38	private
  	39
  	40	def assert_next_token(ts, text, start, _end)
  	41	token = ts.next
  	42	assert_equal token.text, text
  	43	assert_equal token.start, start
  	44	assert_equal token.end, _end
  	45	end
  	46	end
  	47

• /plugins/multilingual_ferret_tools/README

  ......	@@ -0,0 +1,23 @@
  	1	MultilngualFerretTools
  	2	======================
  	3
  	4	This plugin provides a multilingual-aware Analyzer for Ferret. This analyzer is useful when
  	5	fields may contain characters from multiple languages. Fields are broken into latin parts and
  	6	non-latin-parts- the latin parts are then processed with Ferret::Analysis::StandardAnalyzer, and
  	7	the non-latin parts are processed with Ferret::Analysis::RegExpAnalyzer. By default, the RegExpAnalyzer
  	8	used considers each character as a distinct token.
  	9
  	10	If you want to change the configuration of the delegated analyzers, you'll find that initialization
  	11	code in MultilingualFerretTools::Analyzer#token_stream_for.
  	12
  	13	Example Usage
  	14	=============
  	15
  	16	class MyModel < ActiveRecord::Base
  	17	acts_as_ferret(
  	18	{ :fields => { blah blah blah },
  	19	{ :analyzer => MultilingualFerretTools::Analyzer.new }
  	20	)
  	21	end
  	22
  	23
  0	24	\ No newline at end of file

• /plugins/multilingual_ferret_tools/lib/multilingual_chunker.rb

  ......	@@ -0,0 +1,65 @@
  	1	module MultilingualFerretTools
  	2	class Chunker
  	3	def initialize(str, options={})
  	4	@codepoints = str.unpack("U*")
  	5	@u_index = 0
  	6	@s_index = 0
  	7	@options = options.reverse_merge!({ :whitespace => :discard })
  	8	end
  	9
  	10	def next
  	11	return nil if @u_index >= @codepoints.length
  	12	start_u_index = @u_index
  	13	start_s_index = @s_index
  	14	this_classification = Chunker.classify(@codepoints[@u_index])
  	15	while @u_index < @codepoints.length - 1
  	16	current_classification = Chunker.classify(@codepoints[@u_index + 1])
  	17	finished = current_classification != this_classification
  	18	finished = false if @options[:whitespace] == :combine and (
  	19	(current_classification == :latin_whitespace and this_classification == :latin) or
  	20	(current_classification == :latin and this_classification == :latin_whitespace) or
  	21	(current_classification == :non_latin_whitespace and this_classification == :non_latin) or
  	22	(current_classification == :non_latin and this_classification == :non_latin_whitespace)
  	23	)
  	24	break if finished
  	25	this_classification = current_classification unless (current_classification == :latin_whitespace or current_classification == :non_latin_whitespace)
  	26	@u_index += 1
  	27	end
  	28
  	29	@u_index += 1
  	30
  	31	this_string = @codepoints[start_u_index..@u_index - 1].pack("U*")
  	32	@s_index += this_string.length
  	33	((this_classification == :latin_whitespace or this_classification == :non_latin_whitespace) and @options[:whitespace] == :discard) ? self.next : [this_string, this_classification, start_s_index, @s_index - 1]
  	34	end
  	35
  	36	def self.classify(thing)
  	37	if thing.is_a?(String)
  	38	chars = thing.unpack("U*")
  	39
  	40	found_latin = found_nonlatin = found_latin_whitespace = found_nonlatin_whitespace = false
  	41	chars.each do |c|
  	42	classification = classify(c)
  	43	found_latin ||= classification == :latin
  	44	found_nonlatin ||= classification == :non_latin
  	45	found_latin_whitespace ||= classification == :latin_whitespace
  	46	found_nonlatin_whitespace ||= classification == :non_latin_whitespace
  	47	end
  	48
  	49	found_latin_whitespace && !found_nonlatin_whitespace && !found_latin && !found_nonlatin ? :latin_whitespace :
  	50	found_nonlatin_whitespace && !found_latin_whitespace && !found_latin && !found_nonlatin ? :non_latin_whitespace :
  	51	found_latin && !found_nonlatin ? :latin : found_nonlatin && !found_latin ? :non_latin : :mixed
  	52	elsif thing.is_a?(Integer)
  	53	@@LATIN_WHITESPACE.include?(thing) ? :latin_whitespace : @@NON_LATIN_WHITESPACE.include?(thing) ? :non_latin_whitespace : thing < 0x300 ? :latin : :non_latin
  	54	else
  	55	:unknown
  	56	end
  	57	end
  	58
  	59	private
  	60
  	61	@@LATIN_WHITESPACE = " \t".unpack("c*")
  	62	@@NON_LATIN_WHITESPACE = [ 0x3000 ]
  	63	@@WHITESPACE = { :latin =>@@LATIN_WHITESPACE, :non_latin => @@NON_LATIN_WHITESPACE }
  	64	end
  	65	end
  0	66	\ No newline at end of file