[open-ils-commits] [GIT] Evergreen ILS branch master updated. 76b277137691da415d50c7fea15ec0dde4a37157
Evergreen Git
git at git.evergreen-ils.org
Wed Aug 19 15:01:55 EDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 76b277137691da415d50c7fea15ec0dde4a37157 (commit)
via 305aae5b36966e0d9a3ff0cf85b6a05df63bd349 (commit)
via 8f11e930d317e83b42f3dd0adbb47044fd89e962 (commit)
via 49584608efd7d3bd74c7a01301d902787776b5ef (commit)
via e3486397bb616b230da2bb8a458767bfff07eddb (commit)
from d81c0b2d58cbef0d26400098fd681fa04d674fef (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 76b277137691da415d50c7fea15ec0dde4a37157
Author: Jason Stephenson <jstephenson at mvlc.org>
Date: Wed Aug 19 15:01:10 2015 -0400
LP 1438136: Stamping upgrade script.
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 9aff082..d58b922 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0939', :eg_version); -- mrpeters/stompro/mmorgan/bshum
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0940', :eg_version); -- miker/jboyer/csharp/dyrcona
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
similarity index 95%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
rename to Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
index a7b992b..6e55e03 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
@@ -1,6 +1,6 @@
BEGIN;
-SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0939', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
commit 305aae5b36966e0d9a3ff0cf85b6a05df63bd349
Author: Mike Rylander <mrylander at gmail.com>
Date: Sun Apr 26 12:29:18 2015 -0400
LP#1438136: Make upgrade script forgiving
If the functions already exist, that's OK.
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Jason Boyer <jboyer at library.in.gov>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index f405c1a..9fa639f 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -46,7 +46,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index 2b8e842..a7b992b 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -27,7 +27,7 @@ BEGIN
END;
$$ LANGUAGE PLPGSQL;
-CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+CREATE OR REPLACE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
BEGIN
RETURN $1 @@ $2::query_int;
END;
commit 8f11e930d317e83b42f3dd0adbb47044fd89e962
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Apr 17 12:37:06 2015 -0400
LP#1438136: Protect PG < 9.2 from our stats probe
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index ac3174f..84fdbee 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -697,6 +697,7 @@ my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
our %_dfilter_stats_cache = ();
+our $_pg_version = 0;
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -704,11 +705,20 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
- if (!scalar keys %_dfilter_stats_cache) {
- my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
- %_dfilter_stats_cache = map {
- ( $_->{element}, $_->{frequency} )
- } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ if (!$_pg_version) {
+ ($_pg_version = $e->json_query({from => ['version']})->[0]->{version}) =~ s/^.+?(\d\.\d).+$/$1/;
+ }
+
+ my $common = 0;
+ if ($_pg_version >= 9.2) {
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+ } else {
+ $common = 1; # Assume it's expensive
}
if (!exists($_dfilter_controlled_cache{$filter})) {
@@ -727,7 +737,7 @@ sub dynamic_filter_compile {
'code' : 'value';
my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
- my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+ $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects) unless $common;
return (sprintf('%s(%s)', $negate,
join(
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index cb43384..f405c1a 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -23,6 +23,7 @@ CREATE SCHEMA search;
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
index cff061a..2b8e842 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -4,6 +4,7 @@ SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
BEGIN
+ -- This query will die on PG < 9.2, but the function can be created. We just won't use it where we can't.
RETURN QUERY
SELECT e,
f
commit 49584608efd7d3bd74c7a01301d902787776b5ef
Author: Mike Rylander <mrylander at gmail.com>
Date: Thu Apr 16 17:39:49 2015 -0400
LP#1438136: Update baseline schema files
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index 5907e05..cb43384 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -21,6 +21,36 @@ BEGIN;
CREATE SCHEMA search;
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
CREATE TABLE search.relevance_adjustment (
id SERIAL PRIMARY KEY,
active BOOL NOT NULL DEFAULT TRUE,
commit e3486397bb616b230da2bb8a458767bfff07eddb
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Apr 15 12:10:27 2015 -0400
LP#1438136: Teach QP to be smart about attribute selectivity
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Chris Sharp <csharp at georgialibraries.org>
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 147b165..ac3174f 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -696,6 +696,7 @@ use OpenILS::Application::AppUtils;
my $apputils = "OpenILS::Application::AppUtils";
our %_dfilter_controlled_cache = ();
+our %_dfilter_stats_cache = ();
sub dynamic_filter_compile {
my ($self, $filter, $params, $negate) = @_;
@@ -703,6 +704,13 @@ sub dynamic_filter_compile {
$negate = $negate ? '!' : '';
+ if (!scalar keys %_dfilter_stats_cache) {
+ my $data = $e->json_query({from => ['evergreen.pg_statistics', 'record_attr_vector_list', 'vlist']});
+ %_dfilter_stats_cache = map {
+ ( $_->{element}, $_->{frequency} )
+ } grep { $_->{frequency} > 5 } @$data; # Pin floor to 5% of the table
+ }
+
if (!exists($_dfilter_controlled_cache{$filter})) {
my $crad = $e->retrieve_config_record_attr_definition($filter);
my $ccvm_list = $e->search_config_coded_value_map({ctype =>$filter});
@@ -718,16 +726,15 @@ sub dynamic_filter_compile {
my $value_field = $_dfilter_controlled_cache{$filter}{controlled} ?
'code' : 'value';
- return sprintf('%s(%s)', $negate,
+ my $attr_objects = $e->$method({ $attr_field => $filter, $value_field => $params });
+ my $common = scalar(grep { exists($_dfilter_stats_cache{$_->id}) } @$attr_objects);
+
+ return (sprintf('%s(%s)', $negate,
join(
'|',
- map {
- $_->id
- } @{
- $e->$method({ $attr_field => $filter, $value_field => $params })
- }
+ map { $_->id } @$attr_objects
)
- );
+ ), $common);
}
sub toSQL {
@@ -1059,6 +1066,7 @@ sub flatten {
my $joiner = "\n" . ${spc} x ( $self->plan_level + 5 ) . ($self->joiner eq '&' ? 'AND ' : 'OR ');
my @dlist = ();
+ my $common = 0;
# for each dynamic filter, build more of the WHERE clause
for my $filter (@{$self->filters}) {
my $NOT = $filter->negate ? 'NOT ' : '';
@@ -1070,7 +1078,8 @@ sub flatten {
warn "flatten(): processing dynamic filter ". $filter->name ."\n"
if $self->QueryParser->debug;
- my $vlist_query = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
+ my $vlist_query;
+ ($vlist_query, $common) = $self->dynamic_filter_compile( $fname, $filter->args, $filter->negate );
# bool joiner for intra-plan nodes/filters
push(@dlist, $self->joiner) if @dlist;
@@ -1230,10 +1239,17 @@ sub flatten {
if (@dlist) {
$where .= $joiner if $where ne '';
- $where .= sprintf(
- 'mrv.vlist @@ \'%s\'',
- join('', @dlist)
- );
+ if ($common) { # Use a function wrapper to inform PG of the non-rareness of one or more filter elements
+ $where .= sprintf(
+ 'evergreen.query_int_wrapper(mrv.vlist, \'%s\')',
+ join('', @dlist)
+ );
+ } else {
+ $where .= sprintf(
+ 'mrv.vlist @@ \'%s\'',
+ join('', @dlist)
+ );
+ }
}
warn "flatten(): full filter where => $where\n" if $self->QueryParser->debug;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
new file mode 100644
index 0000000..cff061a
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vlist_freq.sql
@@ -0,0 +1,36 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION evergreen.pg_statistics (tab TEXT, col TEXT) RETURNS TABLE(element TEXT, frequency INT) AS $$
+BEGIN
+ RETURN QUERY
+ SELECT e,
+ f
+ FROM (SELECT ROW_NUMBER() OVER (),
+ (f * 100)::INT AS f
+ FROM (SELECT UNNEST(most_common_elem_freqs) AS f
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )x
+ ) AS f
+ JOIN (SELECT ROW_NUMBER() OVER (),
+ e
+ FROM (SELECT UNNEST(most_common_elems::text::text[]) AS e
+ FROM pg_stats
+ WHERE tablename = tab
+ AND attname = col
+ )y
+ ) AS elems USING (row_number);
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE FUNCTION evergreen.query_int_wrapper (INT[],TEXT) RETURNS BOOL AS $$
+BEGIN
+ RETURN $1 @@ $2::query_int;
+END;
+$$ LANGUAGE PLPGSQL STABLE;
+
+COMMIT;
+
-----------------------------------------------------------------------
Summary of changes:
.../Application/Storage/Driver/Pg/QueryParser.pm | 50 +++++++++++++++-----
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/300.schema.staged_search.sql | 31 ++++++++++++
.../src/sql/Pg/upgrade/0940.schema.vlist_freq.sql | 37 ++++++++++++++
4 files changed, 107 insertions(+), 13 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0940.schema.vlist_freq.sql
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list