sergiotarxz/burguillos.info
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding a Stemmer insteado of suffixes.

Browse Source
This commit is contained in:
sergiotarxz 2023-09-05 19:29:30 +02:00
parent 470f0facaa
commit 9f309be709
2 changed files with 11 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Build.PL
View File

@ -25,6 +25,7 @@ my $build = Module::Build->new(
'Moo::Role' => 0, 'Moo::Role' => 0,
'Module::Pluggable' => 0, 'Module::Pluggable' => 0,
'List::AllUtils' => 0, 'List::AllUtils' => 0,
'Lingua::Stem::Snowball' => 0,
}, },
); );
$build->create_build_script; $build->create_build_script;

50
lib/BurguillosInfo/IndexUtils.pm
View File

@ -12,49 +12,19 @@ use Unicode::Normalize qw/NFKD/;
use Moo; use Moo;
sub normalize ( $self, $text ) { use Lingua::Stem::Snowball;
sub normalize($self, $text) {
return undef if !defined $text; return undef if !defined $text;
my $decomposed = NFKD($text); my $decomposed = NFKD($text);
$decomposed =~ s/\p{NonspacingMark}//g; $decomposed =~ s/\p{NonspacingMark}//g;
$decomposed =~ s/es\b//g; my @words;
$decomposed =~ s/as\b//g; while ($decomposed =~ /\b(\w+)\b/g) {
$decomposed =~ s/os\b//g; push @words, $1;
$decomposed =~ s/e\b//g; }
$decomposed =~ s/o\b//g; my $stemmer = Lingua::Stem::Snowball->new( lang => 'es' );
$decomposed =~ s/a\b//g; $stemmer->stem_in_place(\@words);
$decomposed =~ s/i\b//g; $decomposed = join " ", @words;
$decomposed =~ s/cion\b//g;
$decomposed =~ s/diccion\b//g;
$decomposed =~ s/duccion\b//g;
$decomposed =~ s/dur\b//g;
$decomposed =~ s/eccion\b//g;
$decomposed =~ s/epcion\b//g;
$decomposed =~ s/mient\b//g;
$decomposed =~ s/scripcion\b//g;
$decomposed =~ s/sicion\b//g;
$decomposed =~ s/sion\b//g;
$decomposed =~ s/dad\b//g;
$decomposed =~ s/tad\b//g;
$decomposed =~ s/bilidad\b//g;
$decomposed =~ s/edad\b//g;
$decomposed =~ s/idad\b//g;
$decomposed =~ s/ism\b//g;
$decomposed =~ s/ant\b//g;
$decomposed =~ s/ent\b//g;
$decomposed =~ s/dor\b//g;
$decomposed =~ s/der\b//g;
$decomposed =~ s/ist\b//g;
$decomposed =~ s/abl\b//g;
$decomposed =~ s/ant\b//g;
$decomposed =~ s/ent\b//g;
$decomposed =~ s/rgir\b//g;
$decomposed =~ s/ent\b//g;
$decomposed =~ s/errim\b//g;
$decomposed =~ s/ibl\b//g;
$decomposed =~ s/ific\b//g;
$decomposed =~ s/isim\b//g;
$decomposed =~ s/ecer\b//g;
$decomposed =~ s/ific\b//g;
return $decomposed; return $decomposed;
} }