[open-ils-commits] [GIT] Evergreen ILS branch rel_2_7 updated. 813541a513bd8d8ecc2594350d28e1e1acaa06dc
Evergreen Git
git at git.evergreen-ils.org
Wed Aug 19 15:10:21 EDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, rel_2_7 has been updated
via 813541a513bd8d8ecc2594350d28e1e1acaa06dc (commit)
via 5fbf8f14b447bd5708e77b372cdbb5945b1aa945 (commit)
via 451d2ec08645735d0bbcac660bbd6a39eb4d2bb0 (commit)
via fbed5b92f1ffa5cad20e56388e0bdf2c3401abb5 (commit)
via 50e76bbfdbd11ed49156dec36a42884fb6249e2d (commit)
from 88ddb52a04a57f185c7f47025941ca905e386b81 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 813541a513bd8d8ecc2594350d28e1e1acaa06dc
Author: Jason Stephenson <jstephenson at mvlc.org>
Date: Wed Aug 19 15:01:10 2015 -0400
LP 1438136: Stamping upgrade script.
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
Conflicts:
Open-ILS/src/sql/Pg/002.schema.config.sql
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 58d0a8a..919601d 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0936', :eg_version); -- gmcharlt/dyrcona/bshum
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0940', :eg_version); -- miker/jboyer/csharp/dyrcona
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
similarity index 95%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
rename to Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
index a7b992b..6e55e03 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
@@ -1,6 +1,6 @@
BEGIN;
-SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0939', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
commit 5fbf8f14b447bd5708e77b372cdbb5945b1aa945
Author: Mike Rylander <mrylander at gmail.com>
Date: Sun Apr 26 12:29:18 2015 -0400
LP#1438136: Make upgrade script forgiving
If the functions already exist, that's OK.
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Jason Boyer <jboyer at library.in.gov>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index f405c1a..9fa639f 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -46,7 +46,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index 2b8e842..a7b992b 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -27,7 +27,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
commit 451d2ec08645735d0bbcac660bbd6a39eb4d2bb0
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Apr 17 12:37:06 2015 -0400
LP#1438136: Protect PG < 9.2 from our stats probe
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index ac3174f..84fdbee 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -697,6 +697,7 @@ my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
our %_dfilter_stats_cache = ();
+our $_pg_version = 0;
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -704,11 +705,20 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
- if (!scalar keys %_dfilter_stats_cache) {
- my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
- %_dfilter_stats_cache = map {
- ( $_->{element}, $_->{frequency} )
- } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ if (!$_pg_version) {
+ ($_pg_version = $e->json_query({from => ['version']})->[0]->{version}) =~ s/^.+?(\d\.\d).+$/$1/;
+ }
+
+ my $common = 0;
+ if ($_pg_version >= 9.2) {
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+ } else {
+ $common = 1; # Assume it's expensive
}
if (!exists($_dfilter_controlled_cache{$filter})) {
@@ -727,7 +737,7 @@ sub dynamic_filter_compile {
'code' : 'value';
my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
- my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+ $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects) unless $common;
return (sprintf('%s(%s)', $negate,
join(
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index cb43384..f405c1a 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -23,6 +23,7 @@ CREATE SCHEMA search;
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index cff061a..2b8e842 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -4,6 +4,7 @@ SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
commit fbed5b92f1ffa5cad20e56388e0bdf2c3401abb5
Author: Mike Rylander <mrylander at gmail.com>
Date: Thu Apr 16 17:39:49 2015 -0400
LP#1438136: Update baseline schema files
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index 5907e05..cb43384 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -21,6 +21,36 @@ BEGIN;
CREATE SCHEMA search;
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
CREATE TABLE search.relevance_adjustment (
id SERIAL PRIMARY KEY,
active BOOL NOT NULL DEFAULT TRUE,
commit 50e76bbfdbd11ed49156dec36a42884fb6249e2d
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Apr 15 12:10:27 2015 -0400
LP#1438136: Teach QP to be smart about attribute selectivity
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 147b165..ac3174f 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -696,6 +696,7 @@ use OpenILS::Application::AppUtils;
my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
+our %_dfilter_stats_cache = ();
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -703,6 +704,13 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+
if (!exists($_dfilter_controlled_cache{$filter})) {
my $crad = $e->retrieve_config_record_attr_definition($filter);
my $ccvm_list = $e->search_config_coded_value_map({ctype =>$filter});
@@ -718,16 +726,15 @@ sub dynamic_filter_compile {
my $value_field = $_dfilter_controlled_cache{$filter}{controlled} ?
'code' : 'value';
- return sprintf('%s(%s)', $negate,
+ my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
+ my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+
+ return (sprintf('%s(%s)', $negate,
join(
'|',
- map {
- $_->id
- } @{
- $e->$method({ $attr_field => $filter, $value_field => $params })
- }
+ map { $_->id } @$attr_objects
)
- );
+ ), $common);
}
sub toSQL {
@@ -1059,6 +1066,7 @@ sub flatten {
my $joiner = "\n" . ${spc} x ( $self->plan_level + 5 ) . ($self->joiner eq '&' ? 'AND ' : 'OR ');
my @dlist = ();
+ my $common = 0;
# for each dynamic filter, build more of the WHERE clause
for my $filter (@{$self->filters}) {
my $NOT = $filter->negate ? 'NOT ' : '';
@@ -1070,7 +1078,8 @@ sub flatten {
warn "flatten(): processing dynamic filter ". $filter->name ."\n"
if $self->QueryParser->debug;
- my $vlist_query = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
+ my $vlist_query;
+ ($vlist_query, $common) = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
# bool joiner for intra-plan nodes/filters
push(@dlist, $self->joiner) if @dlist;
@@ -1230,10 +1239,17 @@ sub flatten {
if (@dlist) {
$where .= $joiner if $where ne '';
- $where .= sprintf(
- 'mrv.vlist @@ \'%s\'',
- join('', @dlist)
- );
+ if ($common) { # Use a function wrapper to inform PG of the non-rareness of one or more filter elements
+ $where .= sprintf(
+ 'evergreen.query_int_wrapper(mrv.vlist, \'%s\')',
+ join('', @dlist)
+ );
+ } else {
+ $where .= sprintf(
+ 'mrv.vlist @@ \'%s\'',
+ join('', @dlist)
+ );
+ }
}
warn "flatten(): full filter where => $where\n" if $self->QueryParser->debug;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
new file mode 100644
index 0000000..cff061a
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -0,0 +1,36 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
+COMMIT;
+
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 50 +++++++++++++++-----
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/300.schema.staged_search.sql | 31 ++++++++++++
.../src/sql/Pg/upgrade/0940.schema.vlist_freq.sql | 37 ++++++++++++++
4 files changed, 107 insertions(+), 13 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list