[open-ils-commits] [GIT] Evergreen ILS branch rel_2_1 updated. f4addc73b18197c89848a1f946891449d3474558
Evergreen Git
git at git.evergreen-ils.org
Wed Aug 24 14:57:46 EDT 2011
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, rel_2_1 has been updated
via f4addc73b18197c89848a1f946891449d3474558 (commit)
via 219789e063bd14ba9231ba93769b9bf698325351 (commit)
via e5a92f8dbea46e28808e6c811233cabf5a607852 (commit)
via 052ee9e8f93560c92b22b100f66b4dead3fc0c70 (commit)
via e0d7b46deb7ac40b9466f5673cdc68d5bb61b4ea (commit)
from cccb3df5931b4da2284b4fb17c21c1d1be5511cf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit f4addc73b18197c89848a1f946891449d3474558
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 14:36:54 2011 -0400
Protect against div-by-0 for negated words
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index ef966ec..8b66de1 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -488,7 +488,7 @@ sub toSQL {
my $flat_plan = $self->flatten;
# generate the relevance ranking
- my $rel = "AVG(\n\t\t(" . join(")+\n\t\t(", @{$$flat_plan{rank_list}}) . ")\n\t)";
+ my $rel = "AVG(\n\t\t(" . join(")+\n\t\t(", @{$$flat_plan{rank_list}}) . ")\n\t)+1";
# find any supplied sort option
my ($sort_filter) = $self->find_filter('sort');
@@ -661,7 +661,7 @@ sub flatten {
my $table = $node->table;
my $talias = $node->table_alias;
- my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 1.0)";
+ my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 0.0)";
my $core_limit = $self->QueryParser->core_limit || 25000;
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
commit 219789e063bd14ba9231ba93769b9bf698325351
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 13:44:04 2011 -0400
Track count of dummy atoms and use a NULL tsquery when all dummy
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 183f88e..ef966ec 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -667,7 +667,7 @@ sub flatten {
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
$from .= "\n\t\t\tJOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
- if ($node->tsquery) {
+ if ($node->dummy_count < @{$node->only_atoms} ) {
$from .= "\n\t\t\tJOIN (SELECT ". $node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
} else {
$from .= "\n\t\t\t, (SELECT NULL::tsquery AS tsq ) AS x";
@@ -798,8 +798,6 @@ sub sql {
my $self = shift;
my $sql = shift;
- return undef if $self->{dummy};
-
$self->{sql} = $sql if ($sql);
return $self->{sql} if ($self->{sql});
@@ -811,6 +809,8 @@ sub buildSQL {
my $classname = $self->node->classname;
+ return $self->sql("to_tsquery('$classname','')") if $self->{dummy};
+
my $normalizers = $self->node->plan->QueryParser->query_normalizers( $classname );
my $fields = $self->node->fields;
@@ -857,15 +857,23 @@ use base 'QueryParser::query_plan::node';
sub only_atoms {
my $self = shift;
+ $self->{dummy_count} = 0;
+
my $atoms = $self->query_atoms;
my @only_atoms;
for my $a (@$atoms) {
push(@only_atoms, $a) if (ref($a) && $a->isa('QueryParser::query_plan::node::atom'));
+ $self->{dummy_count}++ if (ref($a) && $a->{dummy});
}
return \@only_atoms;
}
+sub dummy_count {
+ my $self = shift;
+ return $self->{dummy_count};
+}
+
sub table {
my $self = shift;
my $table = shift;
@@ -894,9 +902,7 @@ sub tsquery {
for my $atom (@{$self->query_atoms}) {
if (ref($atom)) {
- my $sql = $atom->sql;
- next if !defined($sql);
- $self->{tsquery} .= "\n\t\t\t" .$sql;
+ $self->{tsquery} .= "\n\t\t\t" .$atom->sql;
} else {
$self->{tsquery} .= $atom x 2;
}
commit e5a92f8dbea46e28808e6c811233cabf5a607852
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 11:05:55 2011 -0400
Use unphrases in SQL generation
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index b858fd5..183f88e 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -661,12 +661,17 @@ sub flatten {
my $table = $node->table;
my $talias = $node->table_alias;
- my $node_rank = $node->rank . " * ${talias}.weight";
+ my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 1.0)";
my $core_limit = $self->QueryParser->core_limit || 25000;
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
$from .= "\n\t\t\tJOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
- $from .= "\n\t\t\tJOIN (SELECT ".$node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
+
+ if ($node->tsquery) {
+ $from .= "\n\t\t\tJOIN (SELECT ". $node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
+ } else {
+ $from .= "\n\t\t\t, (SELECT NULL::tsquery AS tsq ) AS x";
+ }
my @bump_fields;
if (@{$node->fields} > 0) {
@@ -701,6 +706,7 @@ sub flatten {
$where .= '(' . $talias . ".id IS NOT NULL";
$where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
+ $where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
$where .= ')';
push @rank_list, $node_rank;
@@ -792,6 +798,8 @@ sub sql {
my $self = shift;
my $sql = shift;
+ return undef if $self->{dummy};
+
$self->{sql} = $sql if ($sql);
return $self->{sql} if ($self->{sql});
@@ -886,7 +894,9 @@ sub tsquery {
for my $atom (@{$self->query_atoms}) {
if (ref($atom)) {
- $self->{tsquery} .= "\n\t\t\t" .$atom->sql;
+ my $sql = $atom->sql;
+ next if !defined($sql);
+ $self->{tsquery} .= "\n\t\t\t" .$sql;
} else {
$self->{tsquery} .= $atom x 2;
}
commit 052ee9e8f93560c92b22b100f66b4dead3fc0c70
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 11:03:34 2011 -0400
add "unphrases" to capture negated phrases ( -"foo bar" ) and make the negator configurable
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index b4dc798..3ccd566 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -13,6 +13,7 @@ our %parser_config = (
group_start => '(',
group_end => ')',
required => '+',
+ disallowed => '-',
modifier => '#'
}
}
@@ -510,7 +511,11 @@ sub decompose {
warn " ** Search class RE: $search_class_re\n" if $self->debug;
my $required_re = $pkg->operator('required');
- $required_re = qr/^\s*\Q$required_re\E/;
+ $required_re = qr/\Q$required_re\E/;
+
+ my $disallowed_re = $pkg->operator('disallowed');
+ $disallowed_re = qr/\Q$disallowed_re\E/;
+
my $and_re = $pkg->operator('and');
$and_re = qr/^\s*\Q$and_re\E/;
@@ -554,7 +559,7 @@ sub decompose {
} elsif ($self->filter_count && /$filter_re/) { # found a filter
warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
$struct->new_filter( $2 => [ split '[,]+', $3 ], $negate );
@@ -562,7 +567,7 @@ sub decompose {
} elsif ($self->filter_count && /$filter_as_class_re/) { # found a filter
warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
$struct->new_filter( $2 => [ split '[,]+', $3 ], $negate );
@@ -620,7 +625,7 @@ sub decompose {
} elsif ($self->facet_class_count && /$facet_re/) { # changing current class
warn "Encountered facet: $1$2 => $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
my $facet = $2;
my $facet_value = [ split '\s*#\s*', $3 ];
$struct->new_facet( $facet => $facet_value, $negate );
@@ -641,28 +646,37 @@ sub decompose {
$_ = $';
$last_type = 'CLASS';
- } elsif (/^\s*"([^"]+)"/) { # phrase, always anded
- warn "Encountered phrase: $1\n" if $self->debug;
+ } elsif (/^\s*($required_re|$disallowed_re)?"([^"]+)"/) { # phrase, always anded
+ warn 'Encountered' . ($1 ? " ['$1' modified]" : '') . " phrase: $2\n" if $self->debug;
$struct->joiner( '&' );
- my $phrase = $1;
+ my $req_ness = $1;
+ my $phrase = $2;
my $class_node = $struct->classed_node($current_class);
- $class_node->add_phrase( $phrase );
- $_ = $phrase . $';
-
- $last_type = '';
- } elsif (/$required_re([^\s)]+)/) { # phrase, always anded
- warn "Encountered required atom (mini phrase): $1\n" if $self->debug;
- my $phrase = $1;
-
- my $class_node = $struct->classed_node($current_class);
- $class_node->add_phrase( $phrase );
+ if ($req_ness eq $pkg->operator('disallowed')) {
+ $class_node->add_dummy_atom( node => $class_node );
+ $class_node->add_unphrase( $phrase );
+ $phrase = '';
+ #$phrase =~ s/(^|\s)\b/$1-/g;
+ } else {
+ $class_node->add_phrase( $phrase );
+ }
$_ = $phrase . $';
- $struct->joiner( '&' );
$last_type = '';
+# } elsif (/^\s*$required_re([^\s"]+)/) { # phrase, always anded
+# warn "Encountered required atom (mini phrase): $1\n" if $self->debug;
+#
+# my $phrase = $1;
+#
+# my $class_node = $struct->classed_node($current_class);
+# $class_node->add_phrase( $phrase );
+# $_ = $phrase . $';
+# $struct->joiner( '&' );
+#
+# $last_type = '';
} elsif (/^\s*([^$group_end\s]+)/o) { # atom
warn "Encountered atom: $1\n" if $self->debug;
warn "Remainder: $'\n" if $self->debug;
@@ -673,12 +687,16 @@ sub decompose {
$_ = $after;
$last_type = '';
- my $negator = ($atom =~ s/^-//o) ? '!' : '';
+ my $class_node = $struct->classed_node($current_class);
+
+ my $prefix = ($atom =~ s/^$disallowed_re//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
- if ($atom ne '' and !grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
- my $class_node = $struct->classed_node($current_class);
- $class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $negator, node => $class_node );
+ if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|','-','+')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
+# $class_node->add_phrase( $atom ) if ($atom =~ s/^$required_re//o);
+# $class_node->add_unphrase( $atom ) if ($prefix eq '!');
+
+ $class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $prefix, node => $class_node );
$struct->joiner( '&' );
}
}
@@ -995,6 +1013,15 @@ sub phrases {
return $self->{phrases};
}
+sub unphrases {
+ my $self = shift;
+ my @phrases = @_;
+
+ $self->{unphrases} ||= [];
+ $self->{unphrases} = \@phrases if (@phrases);
+ return $self->{unphrases};
+}
+
sub add_phrase {
my $self = shift;
my $phrase = shift;
@@ -1004,6 +1031,15 @@ sub add_phrase {
return $self;
}
+sub add_unphrase {
+ my $self = shift;
+ my $phrase = shift;
+
+ push(@{$self->unphrases}, $phrase);
+
+ return $self;
+}
+
sub query_atoms {
my $self = shift;
my @query_atoms = @_;
@@ -1030,6 +1066,18 @@ sub add_fts_atom {
return $self;
}
+sub add_dummy_atom {
+ my $self = shift;
+ my @parts = @_;
+
+ my $atom = $self->new_atom( @parts, dummy => 1 );
+
+ push(@{$self->query_atoms}, $self->plan->joiner) if (@{$self->query_atoms});
+ push(@{$self->query_atoms}, $atom);
+
+ return $self;
+}
+
#-------------------------------
package QueryParser::query_plan::node::atom;
commit e0d7b46deb7ac40b9466f5673cdc68d5bb61b4ea
Author: Mike Rylander <mrylander at gmail.com>
Date: Tue Aug 23 15:03:56 2011 -0400
Ignore empty atoms in query decomposition
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index db9dd98..b4dc798 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -676,7 +676,7 @@ sub decompose {
my $negator = ($atom =~ s/^-//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
- if (!grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
+ if ($atom ne '' and !grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
my $class_node = $struct->classed_node($current_class);
$class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $negator, node => $class_node );
$struct->joiner( '&' );
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 22 ++++-
.../lib/OpenILS/Application/Storage/QueryParser.pm | 92 +++++++++++++++-----
2 files changed, 89 insertions(+), 25 deletions(-)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list