|
In a message dated 4/23/2006 9:23:31 A.M. Eastern Standard Time,
[EMAIL PROTECTED] writes:
> hello ,
> Thanks for the help . > But my problem is that i need to get 3 different hashes for all uni bi > tri grams .. > coz later in my code i have to calculate the value as > > trigram{hello how are} /bigram{hello how} > > so how do i relate the 2 values in the hashes ... > i used a code as : > > my($prev1,$prev2) = ('',''); > while (<>) { > @words = split; > foreach my $word (@words) { > $unigram{$word}++; > $bigram{"$prev1 $word}++; > $trigram{"$prev2 $prev1 $word}++; > $prev2 = $prev1; > $prev1 = $word; > } > } > > but not able to process it further .... hi amit --
below is some code that seems to do what you need - assuming i
understand what
you need! note that the invocation is different from the
previous code i sent
in that there is no __DATA__ section and the test string is taken from the
command line. see example in the code. also, there
is no string normalization step,
so you will have to be careful about the test strings you use.
also note that the code you give above seems to fail in that it generates
spurious
keys with leading spaces when $prev1 or $prev2 or both are the null
string.
an alternate version of the grams() function is given so that you can
compare and
contrast.
as before, i would enjoy knowing how this code works out for
you.
hth -- bill walters
----------------- code begins -------------------------------
use strict;
use warnings; # my $test = 'fee fie foe'; my $test = shift; # get quoted test string from command line # e.g., # perl t_grams1.pl "fee fie foe fum" my ($hr_unigrams, $hr_bigrams, $hr_trigrams) = grams($test);
# printf "%d unigram keys %d bigram keys %d trigram keys
\n",
# map { scalar keys %$_ } $hr_unigrams, $hr_bigrams, $hr_trigrams; # # { local $, = '|'; print '', keys(%$hr_unigrams), "\n"; } # { local $, = '|'; print '', keys(%$hr_bigrams), "\n"; } # { local $, = '|'; print '', keys(%$hr_trigrams), "\n"; } print "\n";
print "``$test'' has... \n"; print_grams('unigram', $hr_unigrams); print_grams('bigram', $hr_bigrams); print_grams('trigram', $hr_trigrams); print "\n"; print_gram_ratios($hr_bigrams, $hr_unigrams); print
"\n";
print_gram_ratios($hr_trigrams, $hr_bigrams); print "\n"; sub print_gram_ratios { my ($hr_dividend_gram, # n_gram to be divided: the
`larger' n_gram
$hr_divisor_gram, # n_gram that will divide: the `smaller' n_gram ) = @_; for my $dividend_key (sort keys %$hr_dividend_gram)
{
for my $divisor_key (sort keys
%$hr_divisor_gram) {
# go to
next divisor unless divisor key string is
present
# somewhere in dividend key string. next unless $dividend_key =~ /$divisor_key/; # print
ratio = n_gram_dividend /
n_gram_divisor
print_gram_ratio($hr_dividend_gram, $dividend_key, $hr_divisor_gram, $divisor_key); }
}
}
sub grams { my ($string) = @_;
my (%unigram, %bigram, %trigram);
my @words = split ' ', $string;
for my $i (0 .. $#words) { # also seems to
work
next unless exists
$words[$i]; # no n-grams unless non-null
string
$unigram{$words[$i]}++; next unless exists
$words[$i+1]; # no bi-, trigrams unless
non-null
$bigram{"$words[$i] $words[$i+1]"}++; next unless exists
$words[$i+2]; # no trigrams unless
non-null
$trigram{"$words[$i] $words[$i+1] $words[$i+2]"}++; }
# my ($prev1, $prev2, $curr) = ('', '',
'');
# # foreach my $word (@words) { # seems to work # # $prev2 = $prev1; # $prev1 = $curr; # $curr = $word; # # $unigram{$curr}++; # current word will always be non-null (???) # # next unless $prev1; # no bi-, trigrams unless non-null string # $bigram{"$prev1 $curr"}++; # # next unless $prev2; # no trigrams unless non-null string # $trigram{"$prev2 $prev1 $curr"}++; # # } return \(%unigram, %bigram, %trigram);
}
# sub grams { # INCORRECT - generates spurious bi- and trigrams # # my ($line) = @_; # # my (%unigram, %bigram, %trigram); # # my @words = split ' ', $line; # # my ($prev1, $prev2) = ('', ''); # # foreach my $word (@words) { # $unigram{$word}++; # $bigram{"$prev1 $word"}++; # $trigram{"$prev2 $prev1 $word"}++; # $prev2 = $prev1; # $prev1 = $word; # } # # # return (\%unigram, \%bigram, \%trigram); # return \(%unigram, %bigram, %trigram); # # } sub print_gram_ratio { my ($hr_dividend_gram, $dividend_key, $hr_divisor_gram,
$divisor_key) = @_;
printf "%f = `%s' (%d) / `%s' (%d)
\n",
$hr_dividend_gram->{$dividend_key} / $hr_divisor_gram->{$divisor_key}, $dividend_key, $hr_dividend_gram->{$dividend_key}, $divisor_key, $hr_divisor_gram ->{$divisor_key}; }
sub print_grams { my ($gram_name, $hr_grams) = @_;
my $grams = scalar keys
%$hr_grams;
my $plural = ($grams != 1) ? 's' : ''; printf "%d %s%s \n", $grams, $gram_name, $plural;
for my $key (sort keys %$hr_grams)
{
printf "%4d %s \n", $hr_grams->{$key}, $key; } }
|
_______________________________________________ ActivePerl mailing list [email protected] To unsubscribe: http://listserv.ActiveState.com/mailman/mysubs
