[open-ils-commits] ***SPAM*** [GIT] Evergreen ILS branch master updated. 149ffb312303594a4667e09abca5c670dd2de954
Evergreen Git
git at git.evergreen-ils.org
Tue Apr 8 17:29:27 EDT 2014
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 149ffb312303594a4667e09abca5c670dd2de954 (commit)
via 9984b11f6766106f6d8c5b2e05331f9cf2dbf571 (commit)
via c964cd4ce0fc97db9b726d6ada4902f6388c255e (commit)
via 1764ceb2f02eed93baa7128cf048f3302afe5f16 (commit)
via 16eec856033ee3f3b7f5b4c73068e52a575be3c2 (commit)
from f23274d750213f8d9332ee5edc443f70b432b1de (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 149ffb312303594a4667e09abca5c670dd2de954
Author: Galen Charlton <gmc at esilibrary.com>
Date: Tue Apr 8 14:15:11 2014 -0700
LP#1303940: pin schema upgrade to 0878
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 8f6de52..e604c76 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0877', :eg_version); -- dbwells/bshum
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0878', :eg_version); -- miker/dbwells/gmcharlt
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql b/Open-ILS/src/sql/Pg/upgrade/0878.schema.browse-ingest-null-protect.sql
similarity index 98%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
rename to Open-ILS/src/sql/Pg/upgrade/0878.schema.browse-ingest-null-protect.sql
index beace90..9b64de0 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0878.schema.browse-ingest-null-protect.sql
@@ -1,6 +1,6 @@
BEGIN;
-SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0878', :eg_version);
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$
DECLARE
commit 9984b11f6766106f6d8c5b2e05331f9cf2dbf571
Author: Galen Charlton <gmc at esilibrary.com>
Date: Tue Apr 8 14:09:46 2014 -0700
LP#1303940: don't attempt to store NULL values in keyword, browse, or facet indexes
This patch implements a suggestion by Dan Wells and builds on Mike
Rylander's patch to skip inserting into field_entry and facet
indexes values that have been normalized to NULL. As a side effect,
the following warning will no longer be logged when attempting
to store such values in the facet index:
WARNING: Use of uninitialized value in subroutine entry at /usr/lib/perl/5.18/Unicode/Normalize.pm line 80.
CONTEXT: PL/Perl function "force_unicode_normal_form"
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index 97750e7..1242102 100644
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -809,6 +809,10 @@ BEGIN
END IF;
FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+
+ -- don't store what has been normalized away
+ CONTINUE WHEN ind_data.value IS NULL;
+
IF ind_data.field < 0 THEN
ind_data.field = -1 * ind_data.field;
END IF;
@@ -826,7 +830,7 @@ BEGIN
-- expensive to add a comparison of index_vector to index_vector
-- to the WHERE clause below.
- CONTINUE WHEN ind_data.value IS NULL OR ind_data.sort_value IS NULL;
+ CONTINUE WHEN ind_data.sort_value IS NULL;
value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field);
SELECT INTO mbe_row * FROM metabib.browse_entry
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
index 4a3759d..beace90 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
@@ -35,6 +35,10 @@ BEGIN
END IF;
FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+
+ -- don't store what has been normalized away
+ CONTINUE WHEN ind_data.value IS NULL;
+
IF ind_data.field < 0 THEN
ind_data.field = -1 * ind_data.field;
END IF;
@@ -52,7 +56,7 @@ BEGIN
-- expensive to add a comparison of index_vector to index_vector
-- to the WHERE clause below.
- CONTINUE WHEN ind_data.value IS NULL OR ind_data.sort_value IS NULL;
+ CONTINUE WHEN ind_data.sort_value IS NULL;
value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field);
SELECT INTO mbe_row * FROM metabib.browse_entry
commit c964cd4ce0fc97db9b726d6ada4902f6388c255e
Author: Mike Rylander <mrylander at gmail.com>
Date: Mon Apr 7 14:03:51 2014 -0400
LP#1303940: Protect against bogus data that can breaking indexing
In at least the case of browse indexing, some MARC contents can
cause us problems when normalization (such as stripping all
punctuation) can leave us with a NULL value for index-intended
data. The patch avoids the specific case of browse data being null
and causing an ingest failure.
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index fccde87..97750e7 100644
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -826,6 +826,8 @@ BEGIN
-- expensive to add a comparison of index_vector to index_vector
-- to the WHERE clause below.
+ CONTINUE WHEN ind_data.value IS NULL OR ind_data.sort_value IS NULL;
+
value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field);
SELECT INTO mbe_row * FROM metabib.browse_entry
WHERE value = value_prepped AND sort_value = ind_data.sort_value;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
new file mode 100644
index 0000000..4a3759d
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.browse-ingest-null-protect.sql
@@ -0,0 +1,103 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$
+DECLARE
+ fclass RECORD;
+ ind_data metabib.field_entry_template%ROWTYPE;
+ mbe_row metabib.browse_entry%ROWTYPE;
+ mbe_id BIGINT;
+ b_skip_facet BOOL;
+ b_skip_browse BOOL;
+ b_skip_search BOOL;
+ value_prepped TEXT;
+BEGIN
+
+ SELECT COALESCE(NULLIF(skip_facet, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_facet_indexing' AND enabled)) INTO b_skip_facet;
+ SELECT COALESCE(NULLIF(skip_browse, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_browse_indexing' AND enabled)) INTO b_skip_browse;
+ SELECT COALESCE(NULLIF(skip_search, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_search_indexing' AND enabled)) INTO b_skip_search;
+
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ IF NOT b_skip_search THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ END IF;
+ IF NOT b_skip_facet THEN
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
+ IF NOT b_skip_browse THEN
+ DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id;
+ END IF;
+ END IF;
+
+ FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+ IF ind_data.field < 0 THEN
+ ind_data.field = -1 * ind_data.field;
+ END IF;
+
+ IF ind_data.facet_field AND NOT b_skip_facet THEN
+ INSERT INTO metabib.facet_entry (field, source, value)
+ VALUES (ind_data.field, ind_data.source, ind_data.value);
+ END IF;
+
+ IF ind_data.browse_field AND NOT b_skip_browse THEN
+ -- A caveat about this SELECT: this should take care of replacing
+ -- old mbe rows when data changes, but not if normalization (by
+ -- which I mean specifically the output of
+ -- evergreen.oils_tsearch2()) changes. It may or may not be
+ -- expensive to add a comparison of index_vector to index_vector
+ -- to the WHERE clause below.
+
+ CONTINUE WHEN ind_data.value IS NULL OR ind_data.sort_value IS NULL;
+
+ value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field);
+ SELECT INTO mbe_row * FROM metabib.browse_entry
+ WHERE value = value_prepped AND sort_value = ind_data.sort_value;
+
+ IF FOUND THEN
+ mbe_id := mbe_row.id;
+ ELSE
+ INSERT INTO metabib.browse_entry
+ ( value, sort_value ) VALUES
+ ( value_prepped, ind_data.sort_value );
+
+ mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS);
+ END IF;
+
+ INSERT INTO metabib.browse_entry_def_map (entry, def, source, authority)
+ VALUES (mbe_id, ind_data.field, ind_data.source, ind_data.authority);
+ END IF;
+
+ IF ind_data.search_field AND NOT b_skip_search THEN
+ -- Avoid inserting duplicate rows
+ EXECUTE 'SELECT 1 FROM metabib.' || ind_data.field_class ||
+ '_field_entry WHERE field = $1 AND source = $2 AND value = $3'
+ INTO mbe_id USING ind_data.field, ind_data.source, ind_data.value;
+ -- RAISE NOTICE 'Search for an already matching row returned %', mbe_id;
+ IF mbe_id IS NULL THEN
+ EXECUTE $$
+ INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
+ VALUES ($$ ||
+ quote_literal(ind_data.field) || $$, $$ ||
+ quote_literal(ind_data.source) || $$, $$ ||
+ quote_literal(ind_data.value) ||
+ $$);$$;
+ END IF;
+ END IF;
+
+ END LOOP;
+
+ IF NOT b_skip_search THEN
+ PERFORM metabib.update_combined_index_vectors(bib_id);
+ END IF;
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;
+
commit 1764ceb2f02eed93baa7128cf048f3302afe5f16
Author: Galen Charlton <gmc at esilibrary.com>
Date: Tue Apr 8 13:38:50 2014 -0700
LP#1303940: improve readability of test case
This patch adds some whitespace and a comment in the MARCXML
to clarify the sort of data that triggers the bug.
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg b/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
index f76e868..a8bb0f9 100644
--- a/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
+++ b/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
@@ -4,7 +4,23 @@ SELECT plan(1);
SELECT lives_ok($DATA$
INSERT INTO biblio.record_entry (id, last_xact_id, marc)
-VALUES (999999998, 'pgtap', '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader>00531nam a2200157 a 4500</leader><controlfield tag="001">879841</controlfield><controlfield tag="003">MAnMC</controlfield><controlfield tag="005">20080729170300.0</controlfield><controlfield tag="008"> t19981999enka 0 eng </controlfield><datafield tag="020" ind1=" " ind2=" "><subfield code="a">1859677126</subfield></datafield><datafield tag="040" ind1=" " ind2=" "><subfield code="d">UtOrBLW</subfield></datafield><datafield tag="100" ind1="1" ind2=" "><subfield code="a">:</subfield></datafield><datafield tag="245" ind1="1" ind2="4"><subfield code="a">The Illustrated hints, tips & household skills :</subfield><subfield code="b">the practical, step-by-step home reference manual /</subfield><subfield co
de="c">publisher, Joanna Lorenz.</subfield></datafield><datafield tag="250" ind1=" " ind2=" "><subfield code="a">updated c2000.</subfield></datafield><datafield tag="260" ind1=" " ind2=" "><subfield code="a">London :Lorenz Books :</subfield><subfield code="b">an imprint of Anness Publishing Ltd.,</subfield><subfield code="c">c1998,1999.</subfield></datafield><datafield tag="300" ind1=" " ind2=" "><subfield code="a">256 p. :</subfield><subfield code="b">col. ill. ;</subfield><subfield code="c">30 cm.</subfield></datafield><datafield tag="901" ind1=" " ind2=" "><subfield code="a">879841</subfield><subfield code="b">AUTOGEN</subfield><subfield code="c">879841</subfield><subfield code="t">biblio</subfield></datafield></record>');
+VALUES (999999998, 'pgtap', '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim">
+ <leader>00531nam a2200157 a 4500</leader>
+ <controlfield tag="001">879841</controlfield>
+ <controlfield tag="003">MAnMC</controlfield>
+ <controlfield tag="005">20080729170300.0</controlfield>
+ <controlfield tag="008"> t19981999enka 0 eng </controlfield>
+ <datafield tag="020" ind1=" " ind2=" "><subfield code="a">1859677126</subfield></datafield>
+ <datafield tag="040" ind1=" " ind2=" "><subfield code="d">UtOrBLW</subfield></datafield>
+ <!-- the subfield $a here normalizes to a null value when using the stock
+ indexing definitions
+ -->
+ <datafield tag="100" ind1="1" ind2=" "><subfield code="a">:</subfield></datafield>
+ <datafield tag="245" ind1="1" ind2="4"><subfield code="a">The Illustrated hints, tips & household skills :</subfield><subfield code="b">the practical, step-by-step home reference manual /</subfield><subfield code="c">publisher, Joanna Lorenz.</subfield></datafield>
+ <datafield tag="250" ind1=" " ind2=" "><subfield code="a">updated c2000.</subfield></datafield>
+ <datafield tag="260" ind1=" " ind2=" "><subfield code="a">London :Lorenz Books :</subfield><subfield code="b">an imprint of Anness Publishing Ltd.,</subfield><subfield code="c">c1998,1999.</subfield></datafield>
+ <datafield tag="300" ind1=" " ind2=" "><subfield code="a">256 p. :</subfield><subfield code="b">col. ill. ;</subfield><subfield code="c">30 cm.</subfield></datafield>
+ <datafield tag="901" ind1=" " ind2=" "><subfield code="a">879841</subfield><subfield code="b">AUTOGEN</subfield><subfield code="c">879841</subfield><subfield code="t">biblio</subfield></datafield></record>');
$DATA$,
'We should not get an error for NULL entry.');
commit 16eec856033ee3f3b7f5b4c73068e52a575be3c2
Author: Jason Stephenson <jstephenson at mvlc.org>
Date: Mon Apr 7 15:41:02 2014 -0400
LP 1303940: pg_tap regression test.
Add a pg_tap test to insert a record that should trigger the problem
fixed by LP 1303940. If the test fails, we have a regression.
Signed-off-by: Jason Stephenson <jstephenson at mvlc.org>
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg b/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
new file mode 100644
index 0000000..f76e868
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
@@ -0,0 +1,11 @@
+BEGIN;
+
+SELECT plan(1);
+
+SELECT lives_ok($DATA$
+INSERT INTO biblio.record_entry (id, last_xact_id, marc)
+VALUES (999999998, 'pgtap', '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader>00531nam a2200157 a 4500</leader><controlfield tag="001">879841</controlfield><controlfield tag="003">MAnMC</controlfield><controlfield tag="005">20080729170300.0</controlfield><controlfield tag="008"> t19981999enka 0 eng </controlfield><datafield tag="020" ind1=" " ind2=" "><subfield code="a">1859677126</subfield></datafield><datafield tag="040" ind1=" " ind2=" "><subfield code="d">UtOrBLW</subfield></datafield><datafield tag="100" ind1="1" ind2=" "><subfield code="a">:</subfield></datafield><datafield tag="245" ind1="1" ind2="4"><subfield code="a">The Illustrated hints, tips & household skills :</subfield><subfield code="b">the practical, step-by-step home reference manual /</subfield><subfield co
de="c">publisher, Joanna Lorenz.</subfield></datafield><datafield tag="250" ind1=" " ind2=" "><subfield code="a">updated c2000.</subfield></datafield><datafield tag="260" ind1=" " ind2=" "><subfield code="a">London :Lorenz Books :</subfield><subfield code="b">an imprint of Anness Publishing Ltd.,</subfield><subfield code="c">c1998,1999.</subfield></datafield><datafield tag="300" ind1=" " ind2=" "><subfield code="a">256 p. :</subfield><subfield code="b">col. ill. ;</subfield><subfield code="c">30 cm.</subfield></datafield><datafield tag="901" ind1=" " ind2=" "><subfield code="a">879841</subfield><subfield code="b">AUTOGEN</subfield><subfield code="c">879841</subfield><subfield code="t">biblio</subfield></datafield></record>');
+$DATA$,
+'We should not get an error for NULL entry.');
+
+ROLLBACK;
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/030.schema.metabib.sql | 6 +++
.../Pg/t/regress/lp1303940-browse-hates-null.pg | 27 ++++++++++++++
... => 0878.schema.browse-ingest-null-protect.sql} | 39 ++++++++++++++------
4 files changed, 62 insertions(+), 12 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/t/regress/lp1303940-browse-hates-null.pg
copy Open-ILS/src/sql/Pg/upgrade/{0762.schema.partitioned-ingest.sql => 0878.schema.browse-ingest-null-protect.sql} (74%)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list