[open-ils-commits] [GIT] Evergreen ILS branch rel_2_8 updated. ce46d2076e42e0d6e4efe3173c36239b77f3b224
Evergreen Git
git at git.evergreen-ils.org
Wed Aug 19 15:10:21 EDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, rel_2_8 has been updated
via ce46d2076e42e0d6e4efe3173c36239b77f3b224 (commit)
via 15f8c703b0886e962b6e9ef153133f2a9de26b6d (commit)
via 4b477943b0f4c0529db84fbe323872d442e15ee2 (commit)
via 65d291ed501bdd475ccd24747deb13146c8512c8 (commit)
via 1ea489eb988771e4d942b72fad5373e735f873e5 (commit)
from 9fdbabbf7eee9e9474ffe1d85d6eda43d46105f2 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit ce46d2076e42e0d6e4efe3173c36239b77f3b224
Author: Jason Stephenson <jstephenson at mvlc.org>
Date: Wed Aug 19 15:01:10 2015 -0400
LP 1438136: Stamping upgrade script.
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
Conflicts:
Open-ILS/src/sql/Pg/002.schema.config.sql
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 58d0a8a..919601d 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0936', :eg_version); -- gmcharlt/dyrcona/bshum
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0940', :eg_version); -- miker/jboyer/csharp/dyrcona
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
similarity index 95%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
rename to Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
index a7b992b..6e55e03 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
@@ -1,6 +1,6 @@
BEGIN;
-SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0939', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
commit 15f8c703b0886e962b6e9ef153133f2a9de26b6d
Author: Mike Rylander <mrylander at gmail.com>
Date: Sun Apr 26 12:29:18 2015 -0400
LP#1438136: Make upgrade script forgiving
If the functions already exist, that's OK.
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Jason Boyer <jboyer at library.in.gov>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index f405c1a..9fa639f 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -46,7 +46,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index 2b8e842..a7b992b 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -27,7 +27,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
commit 4b477943b0f4c0529db84fbe323872d442e15ee2
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Apr 17 12:37:06 2015 -0400
LP#1438136: Protect PG < 9.2 from our stats probe
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index ac3174f..84fdbee 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -697,6 +697,7 @@ my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
our %_dfilter_stats_cache = ();
+our $_pg_version = 0;
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -704,11 +705,20 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
- if (!scalar keys %_dfilter_stats_cache) {
- my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
- %_dfilter_stats_cache = map {
- ( $_->{element}, $_->{frequency} )
- } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ if (!$_pg_version) {
+ ($_pg_version = $e->json_query({from => ['version']})->[0]->{version}) =~ s/^.+?(\d\.\d).+$/$1/;
+ }
+
+ my $common = 0;
+ if ($_pg_version >= 9.2) {
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+ } else {
+ $common = 1; # Assume it's expensive
}
if (!exists($_dfilter_controlled_cache{$filter})) {
@@ -727,7 +737,7 @@ sub dynamic_filter_compile {
'code' : 'value';
my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
- my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+ $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects) unless $common;
return (sprintf('%s(%s)', $negate,
join(
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index cb43384..f405c1a 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -23,6 +23,7 @@ CREATE SCHEMA search;
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index cff061a..2b8e842 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -4,6 +4,7 @@ SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
commit 65d291ed501bdd475ccd24747deb13146c8512c8
Author: Mike Rylander <mrylander at gmail.com>
Date: Thu Apr 16 17:39:49 2015 -0400
LP#1438136: Update baseline schema files
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index 5907e05..cb43384 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -21,6 +21,36 @@ BEGIN;
CREATE SCHEMA search;
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
CREATE TABLE search.relevance_adjustment (
id SERIAL PRIMARY KEY,
active BOOL NOT NULL DEFAULT TRUE,
commit 1ea489eb988771e4d942b72fad5373e735f873e5
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Apr 15 12:10:27 2015 -0400
LP#1438136: Teach QP to be smart about attribute selectivity
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Ben Shum <bshum at biblio.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 147b165..ac3174f 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -696,6 +696,7 @@ use OpenILS::Application::AppUtils;
my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
+our %_dfilter_stats_cache = ();
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -703,6 +704,13 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+
if (!exists($_dfilter_controlled_cache{$filter})) {
my $crad = $e->retrieve_config_record_attr_definition($filter);
my $ccvm_list = $e->search_config_coded_value_map({ctype =>$filter});
@@ -718,16 +726,15 @@ sub dynamic_filter_compile {
my $value_field = $_dfilter_controlled_cache{$filter}{controlled} ?
'code' : 'value';
- return sprintf('%s(%s)', $negate,
+ my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
+ my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+
+ return (sprintf('%s(%s)', $negate,
join(
'|',
- map {
- $_->id
- } @{
- $e->$method({ $attr_field => $filter, $value_field => $params })
- }
+ map { $_->id } @$attr_objects
)
- );
+ ), $common);
}
sub toSQL {
@@ -1059,6 +1066,7 @@ sub flatten {
my $joiner = "\n" . ${spc} x ( $self->plan_level + 5 ) . ($self->joiner eq '&' ? 'AND ' : 'OR ');
my @dlist = ();
+ my $common = 0;
# for each dynamic filter, build more of the WHERE clause
for my $filter (@{$self->filters}) {
my $NOT = $filter->negate ? 'NOT ' : '';
@@ -1070,7 +1078,8 @@ sub flatten {
warn "flatten(): processing dynamic filter ". $filter->name ."\n"
if $self->QueryParser->debug;
- my $vlist_query = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
+ my $vlist_query;
+ ($vlist_query, $common) = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
# bool joiner for intra-plan nodes/filters
push(@dlist, $self->joiner) if @dlist;
@@ -1230,10 +1239,17 @@ sub flatten {
if (@dlist) {
$where .= $joiner if $where ne '';
- $where .= sprintf(
- 'mrv.vlist @@ \'%s\'',
- join('', @dlist)
- );
+ if ($common) { # Use a function wrapper to inform PG of the non-rareness of one or more filter elements
+ $where .= sprintf(
+ 'evergreen.query_int_wrapper(mrv.vlist, \'%s\')',
+ join('', @dlist)
+ );
+ } else {
+ $where .= sprintf(
+ 'mrv.vlist @@ \'%s\'',
+ join('', @dlist)
+ );
+ }
}
warn "flatten(): full filter where => $where\n" if $self->QueryParser->debug;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
new file mode 100644
index 0000000..cff061a
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -0,0 +1,36 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
+COMMIT;
+
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 50 +++++++++++++++-----
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/300.schema.staged_search.sql | 31 ++++++++++++
.../src/sql/Pg/upgrade/0940.schema.vlist_freq.sql | 37 ++++++++++++++
4 files changed, 107 insertions(+), 13 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list