[open-ils-commits] [GIT] Evergreen ILS branch master updated. 0efea6dfefb398283432244e3ab801d5f5e8c3c3
Evergreen Git
git at git.evergreen-ils.org
Wed Aug 24 14:50:07 EDT 2011
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 0efea6dfefb398283432244e3ab801d5f5e8c3c3 (commit)
via b7da3f582a6ec388a312170fcd5ef4123c634735 (commit)
via 0136bdfdc54d3215e62b560032c2af7ce22de009 (commit)
via fd27c98c6158d1dabad47e3fb0911924535c70ec (commit)
via b800e1637f3f29fffed0009721592c7bba05024e (commit)
from 292ad799ed84127026534e351bdd16663f76b1f8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 0efea6dfefb398283432244e3ab801d5f5e8c3c3
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 14:36:54 2011 -0400
Protect against div-by-0 for negated words
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 167aa35..5f8f74a 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -497,7 +497,7 @@ sub toSQL {
my $flat_plan = $self->flatten;
# generate the relevance ranking
- my $rel = "AVG(\n\t\t(" . join(")+\n\t\t(", @{$$flat_plan{rank_list}}) . ")\n\t)";
+ my $rel = "AVG(\n\t\t(" . join(")+\n\t\t(", @{$$flat_plan{rank_list}}) . ")\n\t)+1";
# find any supplied sort option
my ($sort_filter) = $self->find_filter('sort');
@@ -720,7 +720,7 @@ sub flatten {
my $table = $node->table;
my $talias = $node->table_alias;
- my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 1.0)";
+ my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 0.0)";
my $core_limit = $self->QueryParser->core_limit || 25000;
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
commit b7da3f582a6ec388a312170fcd5ef4123c634735
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 13:44:04 2011 -0400
Track count of dummy atoms and use a NULL tsquery when all dummy
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index e108802..167aa35 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -726,7 +726,7 @@ sub flatten {
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
$from .= "\n\t\t\tJOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
- if ($node->tsquery) {
+ if ($node->dummy_count < @{$node->only_atoms} ) {
$from .= "\n\t\t\tJOIN (SELECT ". $node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
} else {
$from .= "\n\t\t\t, (SELECT NULL::tsquery AS tsq ) AS x";
@@ -857,8 +857,6 @@ sub sql {
my $self = shift;
my $sql = shift;
- return undef if $self->{dummy};
-
$self->{sql} = $sql if ($sql);
return $self->{sql} if ($self->{sql});
@@ -870,6 +868,8 @@ sub buildSQL {
my $classname = $self->node->classname;
+ return $self->sql("to_tsquery('$classname','')") if $self->{dummy};
+
my $normalizers = $self->node->plan->QueryParser->query_normalizers( $classname );
my $fields = $self->node->fields;
@@ -919,15 +919,23 @@ use base 'QueryParser::query_plan::node';
sub only_atoms {
my $self = shift;
+ $self->{dummy_count} = 0;
+
my $atoms = $self->query_atoms;
my @only_atoms;
for my $a (@$atoms) {
push(@only_atoms, $a) if (ref($a) && $a->isa('QueryParser::query_plan::node::atom'));
+ $self->{dummy_count}++ if (ref($a) && $a->{dummy});
}
return \@only_atoms;
}
+sub dummy_count {
+ my $self = shift;
+ return $self->{dummy_count};
+}
+
sub table {
my $self = shift;
my $table = shift;
@@ -956,9 +964,7 @@ sub tsquery {
for my $atom (@{$self->query_atoms}) {
if (ref($atom)) {
- my $sql = $atom->sql;
- next if !defined($sql);
- $self->{tsquery} .= "\n\t\t\t" .$sql;
+ $self->{tsquery} .= "\n\t\t\t" .$atom->sql;
} else {
$self->{tsquery} .= $atom x 2;
}
commit 0136bdfdc54d3215e62b560032c2af7ce22de009
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 11:05:55 2011 -0400
Use unphrases in SQL generation
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index e8949ef..e108802 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -720,12 +720,17 @@ sub flatten {
my $table = $node->table;
my $talias = $node->table_alias;
- my $node_rank = $node->rank . " * ${talias}.weight";
+ my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 1.0)";
my $core_limit = $self->QueryParser->core_limit || 25000;
$from .= "\n\tLEFT JOIN (\n\t\tSELECT fe.*, fe_weight.weight, x.tsq /* search */\n\t\t FROM $table AS fe";
$from .= "\n\t\t\tJOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
- $from .= "\n\t\t\tJOIN (SELECT ".$node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
+
+ if ($node->tsquery) {
+ $from .= "\n\t\t\tJOIN (SELECT ". $node->tsquery ." AS tsq ) AS x ON (fe.index_vector @@ x.tsq)";
+ } else {
+ $from .= "\n\t\t\t, (SELECT NULL::tsquery AS tsq ) AS x";
+ }
my @bump_fields;
if (@{$node->fields} > 0) {
@@ -760,6 +765,7 @@ sub flatten {
$where .= '(' . $talias . ".id IS NOT NULL";
$where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
+ $where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
$where .= ')';
push @rank_list, $node_rank;
@@ -851,6 +857,8 @@ sub sql {
my $self = shift;
my $sql = shift;
+ return undef if $self->{dummy};
+
$self->{sql} = $sql if ($sql);
return $self->{sql} if ($self->{sql});
@@ -948,7 +956,9 @@ sub tsquery {
for my $atom (@{$self->query_atoms}) {
if (ref($atom)) {
- $self->{tsquery} .= "\n\t\t\t" .$atom->sql;
+ my $sql = $atom->sql;
+ next if !defined($sql);
+ $self->{tsquery} .= "\n\t\t\t" .$sql;
} else {
$self->{tsquery} .= $atom x 2;
}
commit fd27c98c6158d1dabad47e3fb0911924535c70ec
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Aug 24 11:03:34 2011 -0400
add "unphrases" to capture negated phrases ( -"foo bar" ) and make the negator configurable
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index b4dc798..3ccd566 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -13,6 +13,7 @@ our %parser_config = (
group_start => '(',
group_end => ')',
required => '+',
+ disallowed => '-',
modifier => '#'
}
}
@@ -510,7 +511,11 @@ sub decompose {
warn " ** Search class RE: $search_class_re\n" if $self->debug;
my $required_re = $pkg->operator('required');
- $required_re = qr/^\s*\Q$required_re\E/;
+ $required_re = qr/\Q$required_re\E/;
+
+ my $disallowed_re = $pkg->operator('disallowed');
+ $disallowed_re = qr/\Q$disallowed_re\E/;
+
my $and_re = $pkg->operator('and');
$and_re = qr/^\s*\Q$and_re\E/;
@@ -554,7 +559,7 @@ sub decompose {
} elsif ($self->filter_count && /$filter_re/) { # found a filter
warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
$struct->new_filter( $2 => [ split '[,]+', $3 ], $negate );
@@ -562,7 +567,7 @@ sub decompose {
} elsif ($self->filter_count && /$filter_as_class_re/) { # found a filter
warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
$struct->new_filter( $2 => [ split '[,]+', $3 ], $negate );
@@ -620,7 +625,7 @@ sub decompose {
} elsif ($self->facet_class_count && /$facet_re/) { # changing current class
warn "Encountered facet: $1$2 => $3\n" if $self->debug;
- my $negate = ($1 eq '-') ? 1 : 0;
+ my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
my $facet = $2;
my $facet_value = [ split '\s*#\s*', $3 ];
$struct->new_facet( $facet => $facet_value, $negate );
@@ -641,28 +646,37 @@ sub decompose {
$_ = $';
$last_type = 'CLASS';
- } elsif (/^\s*"([^"]+)"/) { # phrase, always anded
- warn "Encountered phrase: $1\n" if $self->debug;
+ } elsif (/^\s*($required_re|$disallowed_re)?"([^"]+)"/) { # phrase, always anded
+ warn 'Encountered' . ($1 ? " ['$1' modified]" : '') . " phrase: $2\n" if $self->debug;
$struct->joiner( '&' );
- my $phrase = $1;
+ my $req_ness = $1;
+ my $phrase = $2;
my $class_node = $struct->classed_node($current_class);
- $class_node->add_phrase( $phrase );
- $_ = $phrase . $';
-
- $last_type = '';
- } elsif (/$required_re([^\s)]+)/) { # phrase, always anded
- warn "Encountered required atom (mini phrase): $1\n" if $self->debug;
- my $phrase = $1;
-
- my $class_node = $struct->classed_node($current_class);
- $class_node->add_phrase( $phrase );
+ if ($req_ness eq $pkg->operator('disallowed')) {
+ $class_node->add_dummy_atom( node => $class_node );
+ $class_node->add_unphrase( $phrase );
+ $phrase = '';
+ #$phrase =~ s/(^|\s)\b/$1-/g;
+ } else {
+ $class_node->add_phrase( $phrase );
+ }
$_ = $phrase . $';
- $struct->joiner( '&' );
$last_type = '';
+# } elsif (/^\s*$required_re([^\s"]+)/) { # phrase, always anded
+# warn "Encountered required atom (mini phrase): $1\n" if $self->debug;
+#
+# my $phrase = $1;
+#
+# my $class_node = $struct->classed_node($current_class);
+# $class_node->add_phrase( $phrase );
+# $_ = $phrase . $';
+# $struct->joiner( '&' );
+#
+# $last_type = '';
} elsif (/^\s*([^$group_end\s]+)/o) { # atom
warn "Encountered atom: $1\n" if $self->debug;
warn "Remainder: $'\n" if $self->debug;
@@ -673,12 +687,16 @@ sub decompose {
$_ = $after;
$last_type = '';
- my $negator = ($atom =~ s/^-//o) ? '!' : '';
+ my $class_node = $struct->classed_node($current_class);
+
+ my $prefix = ($atom =~ s/^$disallowed_re//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
- if ($atom ne '' and !grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
- my $class_node = $struct->classed_node($current_class);
- $class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $negator, node => $class_node );
+ if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|','-','+')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
+# $class_node->add_phrase( $atom ) if ($atom =~ s/^$required_re//o);
+# $class_node->add_unphrase( $atom ) if ($prefix eq '!');
+
+ $class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $prefix, node => $class_node );
$struct->joiner( '&' );
}
}
@@ -995,6 +1013,15 @@ sub phrases {
return $self->{phrases};
}
+sub unphrases {
+ my $self = shift;
+ my @phrases = @_;
+
+ $self->{unphrases} ||= [];
+ $self->{unphrases} = \@phrases if (@phrases);
+ return $self->{unphrases};
+}
+
sub add_phrase {
my $self = shift;
my $phrase = shift;
@@ -1004,6 +1031,15 @@ sub add_phrase {
return $self;
}
+sub add_unphrase {
+ my $self = shift;
+ my $phrase = shift;
+
+ push(@{$self->unphrases}, $phrase);
+
+ return $self;
+}
+
sub query_atoms {
my $self = shift;
my @query_atoms = @_;
@@ -1030,6 +1066,18 @@ sub add_fts_atom {
return $self;
}
+sub add_dummy_atom {
+ my $self = shift;
+ my @parts = @_;
+
+ my $atom = $self->new_atom( @parts, dummy => 1 );
+
+ push(@{$self->query_atoms}, $self->plan->joiner) if (@{$self->query_atoms});
+ push(@{$self->query_atoms}, $atom);
+
+ return $self;
+}
+
#-------------------------------
package QueryParser::query_plan::node::atom;
commit b800e1637f3f29fffed0009721592c7bba05024e
Author: Mike Rylander <mrylander at gmail.com>
Date: Tue Aug 23 15:03:56 2011 -0400
Ignore empty atoms in query decomposition
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index db9dd98..b4dc798 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -676,7 +676,7 @@ sub decompose {
my $negator = ($atom =~ s/^-//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
- if (!grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
+ if ($atom ne '' and !grep { $atom eq $_ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
my $class_node = $struct->classed_node($current_class);
$class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $negator, node => $class_node );
$struct->joiner( '&' );
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 22 ++++-
.../lib/OpenILS/Application/Storage/QueryParser.pm | 92 +++++++++++++++-----
2 files changed, 89 insertions(+), 25 deletions(-)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list