Rubyチュートリアル ─ 英文小説の最頻出ワードを見つけよう! ─(その4)

wdic = WordDictionary.new(ARGF)
 p wdic.top_by_frequency(20)
#> [["the", 16077], ["of", 9823], ["and", 7482], ["to", 7098], ["in", 4456], ["a", 3841], ["that", 3161], ["was", 3040], ["it", 2919], ["i", 2881], ["her", 2550], ["she", 2313], ["as", 2134], ["you", 2071], ["not", 2057], ["be", 2044], ["is", 2033], ["his", 2009], ["he", 1940], ["for", 1927]]

wdic = WordDictionary.new(ARGF)
 wdic.top_by_frequency(20).each { |word, freq| puts "#{freq}:#{word}" }
#>
 16077:the
 9823:of
 7482:and
 7098:to
 4456:in
 3841:a
 3161:that
 3040:was
 2919:it
 2881:i
 2550:her
 2313:she
 2134:as
 2071:you
 2057:not
 2044:be
 2033:is
 2009:his
 1940:he
 1927:for

def pretty_print(data)
 max_stars = 60
 max_value = data.max_by { |key, val| val }.slice(1)
 data.each do |word, freq|
 stars = "*" * (max_stars * (freq/max_value.to_f)).ceil
 printf "%5d:%-5s %s\n", freq, word, stars
 end
 end
 
 wdic = WordDictionary.new(ARGF)
 pretty_print wdic.top_by_frequency(20)

16077:the ************************************************************
 9823:of *************************************
 7482:and ****************************
 7098:to ***************************
 4456:in *****************
 3841:a ***************
 3161:that ************
 3040:was ************
 2919:it ***********
 2881:i ***********
 2550:her **********
 2313:she *********
 2134:as ********
 2071:you ********
 2057:not ********
 2044:be ********
 2033:is ********
 2009:his ********
 1940:he ********
 1927:for ********

class WordDictionary
 attr_reader :words
 def +(other)
 result = (@words + other.words).join(" ")
 WordDictionary.new(result)
 end
 protected :words
 end

alice = WordDictionary.new('public/alice.txt')
 romeo_juliet = WordDictionary.new('public/romeo_juliet.txt')
 p alice.top_by_length(10) { |val| val > 10 }
 p romeo_juliet.top_by_length(10) { |val| val > 10 }
 p (alice + romeo_juliet).top_by_length(10) { |val| val > 10 }
 #> [["caterpillar", 28, 11], ["everything", 14, 10], ["adventures", 12, 10], ["foundation", 25, 10], ["electronic", 27, 10], ["paragraph", 11, 9], ["anxiously", 14, 9], ["beautiful", 13, 9], ["agreement", 18, 9], ["trademark", 11, 9]]
 #> [["distributed", 18, 11], ["shakespeare", 17, 11], ["electronic", 19, 10], ["therefore", 23, 9], ["gentleman", 14, 9], ["gutenberg", 24, 9], ["gentlemen", 11, 9], ["copyright", 16, 9], ["benvolio", 17, 8], ["daughter", 17, 8]]
Argument has assumed as a text string
 #> [["distribution", 16, 12], ["shakespeare", 18, 11], ["caterpillar", 28, 11], ["opportunity", 11, 11], ["information", 13, 11], ["distributed", 22, 11], ["everything", 15, 10], ["adventures", 12, 10], ["foundation", 25, 10], ["permission", 16, 10]]

class WordDictionary
 attr_reader :words
 def +(other)
 result = (@words + other.words).join(" ")
 WordDictionary.new(result)
 end
 protected :words
 def -(other)
 result = (@words - other.words).join(" ")
 WordDictionary.new(result)
 end
 
 def &(other)
 result = (@words & other.words).join(" ")
 WordDictionary.new(result)
 end
 
 def |(other)
 result = (@words | other.words).join(" ")
 WordDictionary.new(result)
 end
 end

class WordDictionary
 attr_reader :words
 def +(other)
 arithmetics(:+, other)
 end
 
 def -(other)
 arithmetics(:-, other)
 end
 
 def &(other)
 arithmetics(:&, other)
 end
 
 def |(other)
 arithmetics(:|, other)
 end
 
 private
 def arithmetics(op, other)
 result = (@words.send op, other.words).join(" ")
 WordDictionary.new(result)
 end
 end

class WordDictionary
 def select(regexp)
 text =
 @freq_dic.select { |word, freq| word =~ regexp }
 .select { |word, freq| block_given? ? yield(freq) : freq }
 .map { |word, freq| "#{word} " * freq }
 .join(" ")
 WordDictionary.new(text, true)
 end
 def uniq_words(n, *base)
 base.inject(self) { |_self, b| _self - b.select(/./) { |freq| freq >= 10 } }.top_by_frequency(n)
 end
 end

if 0ドル == __FILE__
 base = WordDictionary.new('public/base.txt')
 alice = WordDictionary.new('public/alice.txt')
 p alice.uniq_words(40, base)
 end 
 ># Argument has assumed as a text string
 ># Argument has assumed as a text string
 ># [["alice", 403], ["turtle", 59], ["mock", 56], ["hatter", 56], ["gryphon", 55], ["rabbit", 51], ["got", 45], ["mouse", 44], ["duchess", 42], ["tone", 40], ["dormouse", 40], ["cat", 37], ["hare", 31], ["caterpillar", 28], ["jury", 22], ["sort", 20], ["tea", 19], ["soup", 18], ["bill", 17], ["hastily", 16], ["bit", 16], ["doesn", 16], ["didn", 14], ["dinah", 14], ["anxiously", 14], ["baby", 14], ["footman", 14], ["trying", 14], ["cook", 13], ["wouldn", 13]]

class WordDictionary
 att_reader :words, :name
 def initialize(input, name="none", inner_call=false)
 input = input_to_string(input, inner_call)
 @name = name
 end
 def select(regexp)
 text = @freq_dic.select { |word, freq| word =~ regexp }.select { |word, freq| block_given? ? yield(freq) : freq }.map { |word, freq| "#{word} " * freq }.join(" ")
 WordDictionary.new(text, @name, true)
 end
 
 def size
 @freq_dic.length
 end
 private
 def input_to_string(input, inner_call)
 case input
 when String
 begin
 File.open(input, "r") { |f| return f.read }
 rescue
 puts "Argument has assumed as a text string" unless inner_call
 input
 end
 end
 end
 def arithmetics(op, other)
 result = (@words.send op, other.words).join(" ")
 WordDictionary.new(result, '', true)
 end
 end

require "open-uri"
 module Enumerable
 def take_by(nth)
 sort_by { |elem| yield elem }.slice(0...nth)
 end
 end
 
 class WordDictionary
 include Enumerable
 attr_reader :name, :words
 
 def initialize(input, name= 'none', inner_call=false)
 input = input_to_string(input, inner_call)
 @words = input.downcase.scan(/[a-z']+/)
 @freq_dic = @words.inject(Hash.new(0)) { |dic, word| dic[word] += 1 ; dic }
 @name = name
 end
 
 def each
 @freq_dic.each { |elem| yield elem }
 end
 
 def top_by_frequency(nth, &blk)
 take_by_value(nth, lambda { |v| -v }, &blk)
 end
 
 def bottom_by_frequency(nth, &blk)
 take_by_value(nth, lambda { |v| v }, &blk)
 end
 
 def top_by_length(nth, &blk)
 list = take_by_key(nth, lambda { |word| -word.length }, &blk)
 list.map { |word, freq| [word, freq, word.length] }
 end
 
 def select(regexp)
 text = @freq_dic.select { |word, freq| word =~ regexp }.select { |word, freq| block_given? ? yield(freq) : freq }.map { |word, freq| "#{word} " * freq }.join(" ")
 WordDictionary.new(text, @name, true)
 end
 
 def to_s
 @freq_dic.to_s
 end
 
 def size
 @freq_dic.length
 end
 
 def +(other)
 arithmetics(:+, other)
 end
 
 def -(other)
 arithmetics(:-, other)
 end
 
 def &(other)
 arithmetics(:&, other)
 end
 
 def |(other)
 arithmetics(:|, other)
 end
 
 def uniq_words(nth, *base)
 base.inject(self) { |_self, b| _self - b.select(/./) { |freq| freq >= 10 } }.top_by_frequency(nth)
 end
	
 protected :words 
 private
 def input_to_string(input, inner_call)
 case input
 when /^http/
 begin
 open(input) { |f| return f.read }
 rescue Exception => e
 puts e
 exit
 end
 when String
 begin
 File.open(input, "r") { |f| return f.read }
 rescue
 STDERR.puts "Argument has assumed as a text string" unless inner_call
 input
 end
 when ARGF.class
 input.read
 else
 raise "Wrong argument. ARGF, file or string are acceptable."
 end
 end
 
 def take_by_value(nth, sort_opt, &blk)
 val = lambda { |key, val| val }
 take_by_key_or_val(nth, sort_opt, val, &blk)
 end
 
 def take_by_key(nth, sort_opt, &blk)
 key = lambda { |key, val| key }
 take_by_key_or_val(nth, sort_opt, key, &blk)
 end
 
 def take_by_key_or_val(nth, sort_opt, by)
 @freq_dic.select { |key, val| block_given? ? yield(val) : val }.take_by(nth) { |key, val| sort_opt[by[key, val]] }
 end
 
 def arithmetics(op, other)
 result = (@words.send op, other.words).join(" ")
 WordDictionary.new(result, '', true)
 end
 end
 
 def pretty_print(data)
 max_stars = 60
 max_value = data.max_by { |word, freq| freq }.slice(1)
 data.each do |word, freq|
 stars = "*" * (max_stars * (freq/max_value.to_f)).ceil
 printf "%5d:%-5s %s\n", freq, word, stars
 end
 end
 
 if 0ドル == __FILE__
 base = WordDictionary.new('public/base.txt')
 alice = WordDictionary.new('public/alice.txt', "Alice's Adventures in Wonderland")
 jp_history = WordDictionary.new('public/japanese_history.txt')
 p alice.uniq_words(40, base)
 p jp_history.uniq_words(40, base)
 end

• ← Previous
• Archive
• Next →

100円〜で好評発売中!
M'ELBORNE BOOKS