[open-ils-commits] [GIT] Evergreen ILS branch master updated. b90330614c99c08f67b6e51cdf2aa035edfba66b
Evergreen Git
git at git.evergreen-ils.org
Tue May 22 14:56:06 EDT 2012
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via b90330614c99c08f67b6e51cdf2aa035edfba66b (commit)
via 81dbf4d6a4a2c008cac9ca3fef8b0a8158157794 (commit)
from 130ba6db26674313416a0309a0290fe98ec7404e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit b90330614c99c08f67b6e51cdf2aa035edfba66b
Author: Bill Erickson <berick at esilibrary.com>
Date: Tue May 15 18:00:30 2012 -0400
Query Parser nested dynamic filters
Dynamic filters (from metabib.record_attr) are now parsed along with
search strings and facets as Query Parser descends the parse tree. The
WHERE clause for the query as a whole is now a combination of the main
WHERE and the ON clause for JOINing to metabib.record_attr. This gives
us the ability to create complex boolean nested queries using
SVF/record_attr filters, unlike before where nested filters were either
ignored or always treated as global filters.
For example:
concerto && ( ( item_type(a) subject|topic[Music] ) || item_form(a) )
This query now successfully limits the "concerto" search to results that
have either (item_type(a) AND subject|topic[Music]) OR item_form(a).
Signed-off-by: Bill Erickson <berick at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index a406717..89e3c12 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -520,35 +520,9 @@ sub toSQL {
}
$rel = "1.0/($rel)::NUMERIC";
- my %dyn_filters = ( '' => [] ); # the "catch-all" key
- for my $f ( @{ $self->QueryParser->dynamic_filters } ) {
- my $col = $f;
- $col = 'item_lang' if ($f eq 'language'); #XXX filter aliases would address this ... booo ... later
-
- my ($filter) = $self->find_filter($f);
- if ($filter) {
- my @fargs = @{$filter->args};
-
- if (@fargs > 1 || $filter->negate) {
- my $NOT = $filter->negate ? 'NOT' : '';
- $dyn_filters{$f} = "$NOT( " .
- join(
- " OR ",
- map { "mrd.attrs \@> hstore('$col', " . $self->QueryParser->quote_value($_) . ")" } @fargs
- ) .
- " )";
- } else {
- push(@{$dyn_filters{''}}, "hstore('$col', " . $self->QueryParser->quote_value($fargs[0]) . ")");
- }
- }
- }
-
- my $combined_dyn_filters = '';
- $combined_dyn_filters .= 'AND mrd.attrs @> (' . join(' || ', @{$dyn_filters{''}}) . ') ' if (@{$dyn_filters{''}});
- delete($dyn_filters{''});
-
- my @dyn_filter_list = values(%dyn_filters);
- $combined_dyn_filters .= 'AND ' . join(' AND ', @dyn_filter_list) if (@dyn_filter_list);
+ my $mra_join = 'INNER JOIN metabib.record_attr mrd ON (m.source = mrd.id';
+ $mra_join .= ' AND '. $flat_plan->{fwhere} if $flat_plan->{fwhere};
+ $mra_join .= ')';
my $rank = $rel;
@@ -676,16 +650,15 @@ SELECT $key AS id,
$rank AS rank,
FIRST(mrd.attrs->'date1') AS tie_break
FROM metabib.metarecord_source_map m
- JOIN metabib.record_attr mrd ON (m.source = mrd.id)
$container
$record_list
$$flat_plan{from}
+ $mra_join
WHERE 1=1
$before
$after
$during
$between
- $combined_dyn_filters
$flat_where
GROUP BY 1
ORDER BY 4 $desc $nullpos, 5 DESC $nullpos, 3 DESC
@@ -726,9 +699,18 @@ sub flatten {
my $from = shift || '';
my $where = shift || '(';
my $with = '';
+ my $fwhere = shift || ''; # for joining dynamic filters (mra)
+
+ my @dyn_filters;
+ for my $filter (@{$self->filters}) {
+ push(@dyn_filters, $filter) if
+ grep { $_ eq $filter->name }
+ @{ $self->QueryParser->dynamic_filters };
+ };
my @rank_list;
for my $node ( @{$self->query_nodes} ) {
+
if (ref($node)) {
if ($node->isa( 'QueryParser::query_plan::node' )) {
@@ -794,10 +776,19 @@ sub flatten {
}
}
- $where .= '(' . $talias . ".id IS NOT NULL";
- $where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
- $where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
- $where .= ')';
+
+ my $twhere .= '(' . $talias . ".id IS NOT NULL";
+ $twhere .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
+ $twhere .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
+ $twhere .= ')';
+
+ if (@dyn_filters or !$self->top_plan) {
+ # if this WHERE is represented within the dynamic
+ # filter's ON clause, it's not also needed in the main WHERE.
+ $fwhere .= $twhere;
+ } else {
+ $where .= $twhere;
+ }
push @rank_list, $node_rank;
@@ -813,19 +804,35 @@ sub flatten {
@field_ids = @{ $self->QueryParser->facet_field_ids_by_class( $node->classname ) };
}
- my $join_type = $node->negate ? 'LEFT' : 'INNER';
+ my $join_type = ($node->negate or @dyn_filters or !$self->top_plan) ? 'LEFT' : 'INNER';
$from .= "\n${spc}$join_type JOIN /* facet */ metabib.facet_entry $talias ON (\n${spc}${spc}m.source = ${talias}.source\n${spc}${spc}".
"AND SUBSTRING(${talias}.value,1,1024) IN (" . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n${spc}${spc}".
"AND ${talias}.field IN (". join(',', @field_ids) . ")\n${spc})";
- $where .= $node->negate ? "${talias}.id IS NULL" : 'TRUE';
+ if (@dyn_filters or !$self->top_plan) {
+ my $NOT = $node->negate ? '' : ' NOT';
+ $fwhere .= "${talias}.id IS$NOT NULL";
+ } else {
+ $where .= $node->negate ? "${talias}.id IS NULL" : 'TRUE';
+ }
} else {
my $subnode = $node->flatten;
+ # strip the trailing bool from the previous loop if there is
+ # nothing to add to the where/fwhere within this loop.
+ if ($$subnode{where} eq '()') {
+ $where =~ s/\s(AND|OR)\s$//;
+
+ } elsif ($$subnode{fwhere} eq '') {
+ $fwhere =~ s/\s(AND|OR)\s$//;
+ }
+
push(@rank_list, @{$$subnode{rank_list}});
$from .= $$subnode{from};
- $where .= "($$subnode{where})";
+
+ $where .= "($$subnode{where})" unless $$subnode{where} eq '()';
+ $fwhere .= "($$subnode{fwhere})" if $$subnode{fwhere};
if ($$subnode{with}) {
$with .= ', ' if $with;
@@ -833,14 +840,47 @@ sub flatten {
}
}
} else {
- $where .= ' AND ' if ($node eq '&');
- $where .= ' OR ' if ($node eq '|');
- # ... stitching the WHERE together ...
+
+ warn "flatten(): appending WHERE bool to: $where\n" if $self->QueryParser->debug;
+
+ if ($fwhere) {
+ # bool joiner for inter-plan filters
+ $fwhere .= ' AND ' if ($node eq '&');
+ $fwhere .= ' OR ' if ($node eq '|');
+
+ } elsif ($where ne '(') {
+
+ $where .= ' AND ' if ($node eq '&');
+ $where .= ' OR ' if ($node eq '|');
+ }
}
}
- return { rank_list => \@rank_list, from => $from, where => $where.')', with => $with };
+ # for each dynamic filter, build the ON clause for the JOIN
+ for my $filter (@dyn_filters) {
+
+ warn "flatten(): processing dynamic filter ". $filter->name ."\n"
+ if $self->QueryParser->debug;
+
+ # bool joiner for intra-plan nodes/filters
+ $fwhere .= sprintf(" %s ", ($self->joiner eq '&' ? 'AND' : 'OR')) if $fwhere;
+
+ my @fargs = @{$filter->args};
+ my $NOT = $filter->negate ? ' NOT' : '';
+ my $fname = $filter->name;
+ $fname = 'item_lang' if $fname eq 'language'; #XXX filter aliases
+
+ $fwhere .= sprintf(
+ "attrs->'%s'$NOT IN (%s)", $fname,
+ join(',', map { $self->QueryParser->quote_value($_) } @fargs)
+ );
+
+ warn "flatten(): filter where => $fwhere\n"
+ if $self->QueryParser->debug;
+ }
+ warn "flatten(): full filter where => $fwhere\n" if $self->QueryParser->debug;
+ return { rank_list => \@rank_list, from => $from, where => $where.')', with => $with, fwhere => $fwhere };
}
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 07d19c0..d5e12a4 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -723,7 +723,10 @@ sub decompose {
}
- $struct = undef if (scalar(@{$struct->query_nodes}) == 0 && !$struct->top_plan);
+ $struct = undef if
+ scalar(@{$struct->query_nodes}) == 0 &&
+ scalar(@{$struct->filters}) == 0 &&
+ !$struct->top_plan;
return $struct if !wantarray;
return ($struct, $remainder);
commit 81dbf4d6a4a2c008cac9ca3fef8b0a8158157794
Author: Bill Erickson <berick at esilibrary.com>
Date: Tue May 15 17:59:56 2012 -0400
QueryParser test script
Signed-off-by: Bill Erickson <berick at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/support-scripts/test-scripts/query_parser.pl b/Open-ILS/src/support-scripts/test-scripts/query_parser.pl
new file mode 100755
index 0000000..3c647da
--- /dev/null
+++ b/Open-ILS/src/support-scripts/test-scripts/query_parser.pl
@@ -0,0 +1,105 @@
+#!/usr/bin/perl
+require '../oils_header.pl';
+use warnings;
+use strict;
+use OpenILS::Application::Storage::Driver::Pg::QueryParser;
+use JSON::XS;
+use Getopt::Long;
+use Data::Dumper;
+$Data::Dumper::Indent = 1;
+use Time::HiRes qw/time/;
+use OpenILS::Utils::CStoreEditor;
+
+OpenILS::Application::Storage::Driver::Pg::QueryParser->TEST_SETUP;
+
+my $query = '#available title: foo bar* || (-baz || (subject:"1900'.
+ '-1910 junk" "and another thing" se:stuff #available '.
+ 'statuses(0,7,12))) && && && au:malarky || au|'.
+ 'corporate|personal:gonzo && dc.identifier:+123456789X'.
+ ' dc.contributor=rowling #metarecord estimation_'.
+ 'strategy(exclusion) item_type(a, t) item_form(d) '.
+ 'bib.subjectTitle=potter bib.subjectName=harry '.
+ 'keyword|mapscale:1:250000';
+
+#$query = 'concerto #available filter_group_entry(1,2,3) filter_group_entry(4,5)';
+#$query = 'concerto || filter_group_entry(4) || filter_group_entry(3)';
+#$query = 'concerto (audience(a) || (item_type(a) && item_form(b)))';
+#$query = 'concerto || (piano && (item_type(a) || audience(a)))';
+
+#$query = '(concerto item_type(a)) || (piano item_type(b))';
+#$query = 'audience(a) (concerto || item_type(a) || (piano music item_form(b)))';
+#$query = 'concerto && (item_type(a) || piano) && (item_form(b) || music)';
+$query = 'concerto && (piano || item_type(a)) && (music || item_form(b))';
+
+my $superpage = 1;
+my $superpage_size = 1000;
+my $core_limit = 25000;
+my $debug;
+my $config = '/openils/conf/opensrf_core.xml';
+my $quiet = 0;
+
+GetOptions(
+ 'superpage=i' => \$superpage,
+ 'superpage-size=i' => \$superpage_size,
+ 'core-limit=i' => \$core_limit,
+ 'query=s' => \$query,
+ 'debug' => \$debug,
+ 'quiet' => \$quiet,
+ 'config=s' => \$config
+);
+
+osrf_connect($config);
+
+my $parser = OpenILS::Application::Storage::Driver::Pg::QueryParser->new(
+ superpage_size => $superpage_size,
+ superpage => $superpage,
+ core_limit => $core_limit,
+ query => $query,
+ debug => $debug
+);
+
+# load the parser config
+my $cstore = OpenSRF::AppSession->create( 'open-ils.cstore' );
+$parser->initialize(
+ config_record_attr_index_norm_map =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.record_attr_index_norm_map.search.atomic',
+ { id => { "!=" => undef } },
+ { flesh => 1, flesh_fields => { crainm => [qw/norm/] }, order_by => [{ class => "crainm", field => "pos" }] }
+ )->gather(1),
+ search_relevance_adjustment =>
+ $cstore->request(
+ 'open-ils.cstore.direct.search.relevance_adjustment.search.atomic',
+ { id => { "!=" => undef } }
+ )->gather(1),
+ config_metabib_field =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_field.search.atomic',
+ { id => { "!=" => undef } }
+ )->gather(1),
+ config_metabib_search_alias =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_search_alias.search.atomic',
+ { alias => { "!=" => undef } }
+ )->gather(1),
+ config_metabib_field_index_norm_map =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_field_index_norm_map.search.atomic',
+ { id => { "!=" => undef } },
+ { flesh => 1, flesh_fields => { cmfinm => [qw/norm/] }, order_by => [{ class => "cmfinm", field => "pos" }] }
+ )->gather(1),
+ config_record_attr_definition =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.record_attr_definition.search.atomic',
+ { name => { "!=" => undef } }
+ )->gather(1),
+);
+
+$parser->parse;
+
+print "Parsed query tree:\n" . Dumper($parser->parse_tree) unless $quiet;
+
+my $sql = $parser->toSQL;
+$sql =~ s/^\s*$//gm;
+print "SQL:\n$sql\n\n" unless $quiet;
+
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 124 +++++++++++++-------
.../lib/OpenILS/Application/Storage/QueryParser.pm | 5 +-
.../support-scripts/test-scripts/query_parser.pl | 105 +++++++++++++++++
3 files changed, 191 insertions(+), 43 deletions(-)
create mode 100755 Open-ILS/src/support-scripts/test-scripts/query_parser.pl
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list