[open-ils-commits] [GIT] Evergreen ILS branch master updated. 4b5b791fffef5420e6d142964dfefdb647051e22
Evergreen Git
git at git.evergreen-ils.org
Tue Aug 9 12:05:04 EDT 2016
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 4b5b791fffef5420e6d142964dfefdb647051e22 (commit)
via 982472f1110b5874757d57c421e121c0a5962f61 (commit)
via f668201078a11fe06d9bb8e1c658648e7ed2be60 (commit)
via 6cc1d1e0911f4dd4a92d17a9b60025888b080bff (commit)
via cf3bd149e53585676a517d38093f2470dbcfaaeb (commit)
from b55b3cafbb9c64c39b1ad1fbbc0074066d616bba (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 4b5b791fffef5420e6d142964dfefdb647051e22
Author: Mike Rylander <mrylander at gmail.com>
Date: Tue Aug 9 12:04:35 2016 -0400
Stamping upgrade for reingest speedup
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 4e37f2c..27f5d85 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0984', :eg_version); -- miker/gmcharlt/kmlussier
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0985', :eg_version); -- dbwells/gmcharlt/miker
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql b/Open-ILS/src/sql/Pg/upgrade/0985.schema.speed_record_attr_ingest.sql
similarity index 99%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql
rename to Open-ILS/src/sql/Pg/upgrade/0985.schema.speed_record_attr_ingest.sql
index c4e2798..1d38f7d 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0985.schema.speed_record_attr_ingest.sql
@@ -1,6 +1,6 @@
BEGIN;
--- SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0985', :eg_version);
CREATE OR REPLACE FUNCTION metabib.reingest_record_attributes (rid BIGINT, pattr_list TEXT[] DEFAULT NULL, prmarc TEXT DEFAULT NULL, rdeleted BOOL DEFAULT TRUE) RETURNS VOID AS $func$
DECLARE
commit 982472f1110b5874757d57c421e121c0a5962f61
Author: Galen Charlton <gmc at esilibrary.com>
Date: Fri Jul 29 11:40:46 2016 -0400
LP#1588543: schema update script
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql
new file mode 100644
index 0000000..c4e2798
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.speed_record_attr_ingest.sql
@@ -0,0 +1,231 @@
+BEGIN;
+
+-- SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION metabib.reingest_record_attributes (rid BIGINT, pattr_list TEXT[] DEFAULT NULL, prmarc TEXT DEFAULT NULL, rdeleted BOOL DEFAULT TRUE) RETURNS VOID AS $func$
+DECLARE
+ transformed_xml TEXT;
+ rmarc TEXT := prmarc;
+ tmp_val TEXT;
+ prev_xfrm TEXT;
+ normalizer RECORD;
+ xfrm config.xml_transform%ROWTYPE;
+ attr_vector INT[] := '{}'::INT[];
+ attr_vector_tmp INT[];
+ attr_list TEXT[] := pattr_list;
+ attr_value TEXT[];
+ norm_attr_value TEXT[];
+ tmp_xml TEXT;
+ attr_def config.record_attr_definition%ROWTYPE;
+ ccvm_row config.coded_value_map%ROWTYPE;
+BEGIN
+
+ IF attr_list IS NULL OR rdeleted THEN -- need to do the full dance on INSERT or undelete
+ SELECT ARRAY_AGG(name) INTO attr_list FROM config.record_attr_definition
+ WHERE (
+ tag IS NOT NULL OR
+ fixed_field IS NOT NULL OR
+ xpath IS NOT NULL OR
+ phys_char_sf IS NOT NULL OR
+ composite
+ ) AND (
+ filter OR sorter
+ );
+ END IF;
+
+ IF rmarc IS NULL THEN
+ SELECT marc INTO rmarc FROM biblio.record_entry WHERE id = rid;
+ END IF;
+
+ FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE NOT composite AND name = ANY( attr_list ) ORDER BY format LOOP
+
+ attr_value := '{}'::TEXT[];
+ norm_attr_value := '{}'::TEXT[];
+ attr_vector_tmp := '{}'::INT[];
+
+ SELECT * INTO ccvm_row FROM config.coded_value_map c WHERE c.ctype = attr_def.name LIMIT 1;
+
+ -- tag+sf attrs only support SVF
+ IF attr_def.tag IS NOT NULL THEN -- tag (and optional subfield list) selection
+ SELECT ARRAY[ARRAY_TO_STRING(ARRAY_AGG(value), COALESCE(attr_def.joiner,' '))] INTO attr_value
+ FROM (SELECT * FROM metabib.full_rec ORDER BY tag, subfield) AS x
+ WHERE record = rid
+ AND tag LIKE attr_def.tag
+ AND CASE
+ WHEN attr_def.sf_list IS NOT NULL
+ THEN POSITION(subfield IN attr_def.sf_list) > 0
+ ELSE TRUE
+ END
+ GROUP BY tag
+ ORDER BY tag
+ LIMIT 1;
+
+ ELSIF attr_def.fixed_field IS NOT NULL THEN -- a named fixed field, see config.marc21_ff_pos_map.fixed_field
+ attr_value := vandelay.marc21_extract_fixed_field_list(rmarc, attr_def.fixed_field);
+
+ IF NOT attr_def.multi THEN
+ attr_value := ARRAY[attr_value[1]];
+ END IF;
+
+ ELSIF attr_def.xpath IS NOT NULL THEN -- and xpath expression
+
+ SELECT INTO xfrm * FROM config.xml_transform WHERE name = attr_def.format;
+
+ -- See if we can skip the XSLT ... it's expensive
+ IF prev_xfrm IS NULL OR prev_xfrm <> xfrm.name THEN
+ -- Can't skip the transform
+ IF xfrm.xslt <> '---' THEN
+ transformed_xml := oils_xslt_process(rmarc,xfrm.xslt);
+ ELSE
+ transformed_xml := rmarc;
+ END IF;
+
+ prev_xfrm := xfrm.name;
+ END IF;
+
+ IF xfrm.name IS NULL THEN
+ -- just grab the marcxml (empty) transform
+ SELECT INTO xfrm * FROM config.xml_transform WHERE xslt = '---' LIMIT 1;
+ prev_xfrm := xfrm.name;
+ END IF;
+
+ FOR tmp_xml IN SELECT UNNEST(oils_xpath(attr_def.xpath, transformed_xml, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]])) LOOP
+ tmp_val := oils_xpath_string(
+ '//*',
+ tmp_xml,
+ COALESCE(attr_def.joiner,' '),
+ ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]]
+ );
+ IF tmp_val IS NOT NULL AND BTRIM(tmp_val) <> '' THEN
+ attr_value := attr_value || tmp_val;
+ EXIT WHEN NOT attr_def.multi;
+ END IF;
+ END LOOP;
+
+ ELSIF attr_def.phys_char_sf IS NOT NULL THEN -- a named Physical Characteristic, see config.marc21_physical_characteristic_*_map
+ SELECT ARRAY_AGG(m.value) INTO attr_value
+ FROM vandelay.marc21_physical_characteristics(rmarc) v
+ LEFT JOIN config.marc21_physical_characteristic_value_map m ON (m.id = v.value)
+ WHERE v.subfield = attr_def.phys_char_sf AND (m.value IS NOT NULL AND BTRIM(m.value) <> '')
+ AND ( ccvm_row.id IS NULL OR ( ccvm_row.id IS NOT NULL AND v.id IS NOT NULL) );
+
+ IF NOT attr_def.multi THEN
+ attr_value := ARRAY[attr_value[1]];
+ END IF;
+
+ END IF;
+
+ -- apply index normalizers to attr_value
+ FOR tmp_val IN SELECT value FROM UNNEST(attr_value) x(value) LOOP
+ FOR normalizer IN
+ SELECT n.func AS func,
+ n.param_count AS param_count,
+ m.params AS params
+ FROM config.index_normalizer n
+ JOIN config.record_attr_index_norm_map m ON (m.norm = n.id)
+ WHERE attr = attr_def.name
+ ORDER BY m.pos LOOP
+ EXECUTE 'SELECT ' || normalizer.func || '(' ||
+ COALESCE( quote_literal( tmp_val ), 'NULL' ) ||
+ CASE
+ WHEN normalizer.param_count > 0
+ THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
+ ELSE ''
+ END ||
+ ')' INTO tmp_val;
+
+ END LOOP;
+ IF tmp_val IS NOT NULL AND tmp_val <> '' THEN
+ -- note that a string that contains only blanks
+ -- is a valid value for some attributes
+ norm_attr_value := norm_attr_value || tmp_val;
+ END IF;
+ END LOOP;
+
+ IF attr_def.filter THEN
+ -- Create unknown uncontrolled values and find the IDs of the values
+ IF ccvm_row.id IS NULL THEN
+ FOR tmp_val IN SELECT value FROM UNNEST(norm_attr_value) x(value) LOOP
+ IF tmp_val IS NOT NULL AND BTRIM(tmp_val) <> '' THEN
+ BEGIN -- use subtransaction to isolate unique constraint violations
+ INSERT INTO metabib.uncontrolled_record_attr_value ( attr, value ) VALUES ( attr_def.name, tmp_val );
+ EXCEPTION WHEN unique_violation THEN END;
+ END IF;
+ END LOOP;
+
+ SELECT ARRAY_AGG(id) INTO attr_vector_tmp FROM metabib.uncontrolled_record_attr_value WHERE attr = attr_def.name AND value = ANY( norm_attr_value );
+ ELSE
+ SELECT ARRAY_AGG(id) INTO attr_vector_tmp FROM config.coded_value_map WHERE ctype = attr_def.name AND code = ANY( norm_attr_value );
+ END IF;
+
+ -- Add the new value to the vector
+ attr_vector := attr_vector || attr_vector_tmp;
+ END IF;
+
+ IF attr_def.sorter THEN
+ DELETE FROM metabib.record_sorter WHERE source = rid AND attr = attr_def.name;
+ IF norm_attr_value[1] IS NOT NULL THEN
+ INSERT INTO metabib.record_sorter (source, attr, value) VALUES (rid, attr_def.name, norm_attr_value[1]);
+ END IF;
+ END IF;
+
+ END LOOP;
+
+/* We may need to rewrite the vlist to contain
+ the intersection of new values for requested
+ attrs and old values for ignored attrs. To
+ do this, we take the old attr vlist and
+ subtract any values that are valid for the
+ requested attrs, and then add back the new
+ set of attr values. */
+
+ IF ARRAY_LENGTH(pattr_list, 1) > 0 THEN
+ SELECT vlist INTO attr_vector_tmp FROM metabib.record_attr_vector_list WHERE source = rid;
+ SELECT attr_vector_tmp - ARRAY_AGG(id::INT) INTO attr_vector_tmp FROM metabib.full_attr_id_map WHERE attr = ANY (pattr_list);
+ attr_vector := attr_vector || attr_vector_tmp;
+ END IF;
+
+ -- On to composite attributes, now that the record attrs have been pulled. Processed in name order, so later composite
+ -- attributes can depend on earlier ones.
+ PERFORM metabib.compile_composite_attr_cache_init();
+ FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP
+
+ FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP
+
+ tmp_val := metabib.compile_composite_attr( ccvm_row.id );
+ CONTINUE WHEN tmp_val IS NULL OR tmp_val = ''; -- nothing to do
+
+ IF attr_def.filter THEN
+ IF attr_vector @@ tmp_val::query_int THEN
+ attr_vector = attr_vector + intset(ccvm_row.id);
+ EXIT WHEN NOT attr_def.multi;
+ END IF;
+ END IF;
+
+ IF attr_def.sorter THEN
+ IF attr_vector @@ tmp_val THEN
+ DELETE FROM metabib.record_sorter WHERE source = rid AND attr = attr_def.name;
+ INSERT INTO metabib.record_sorter (source, attr, value) VALUES (rid, attr_def.name, ccvm_row.code);
+ END IF;
+ END IF;
+
+ END LOOP;
+
+ END LOOP;
+
+ IF ARRAY_LENGTH(attr_vector, 1) > 0 THEN
+ IF rdeleted THEN -- initial insert OR revivication
+ DELETE FROM metabib.record_attr_vector_list WHERE source = rid;
+ INSERT INTO metabib.record_attr_vector_list (source, vlist) VALUES (rid, attr_vector);
+ ELSE
+ UPDATE metabib.record_attr_vector_list SET vlist = attr_vector WHERE source = rid;
+ END IF;
+ END IF;
+
+END;
+
+$func$ LANGUAGE PLPGSQL;
+
+CREATE INDEX config_coded_value_map_ctype_idx ON config.coded_value_map (ctype);
+
+COMMIT;
commit f668201078a11fe06d9bb8e1c658648e7ed2be60
Author: Galen Charlton <gmc at esilibrary.com>
Date: Fri Jul 29 11:34:06 2016 -0400
LP#1588543: tests for verifying correct generated of record attributes
This patch adds tests to verify that the expected
set of record attributes are generated when a bib record
is ingested. The tests serve a couple purposes:
- verifying that changes to the record attribute ingest
stored procedure don't have unexpected side-effects
- validating record attribute definitions in the seed data
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/t/lp1588543_marc_record_attributes.pg b/Open-ILS/src/sql/Pg/t/lp1588543_marc_record_attributes.pg
new file mode 100644
index 0000000..971da52
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/t/lp1588543_marc_record_attributes.pg
@@ -0,0 +1,211 @@
+-- tests to verify that the expected record attributes are
+-- parsed during record ingest; note that this is expected
+-- to fluctuate as seed data for record attributes definitions
+-- get updated.
+BEGIN;
+
+SELECT plan(2);
+
+INSERT INTO biblio.record_entry (last_xact_id, marc) VALUES (
+ 'marc-attr-test',
+ $record$<record xmlns="http://www.loc.gov/MARC21/slim">
+ <leader>02137cam a2200457 a 4500</leader>
+ <controlfield tag="001">ocn694080497</controlfield>
+ <controlfield tag="003">OCoLC</controlfield>
+ <controlfield tag="005">20160729104757.0</controlfield>
+ <controlfield tag="008">101217s2011 txu b 001 0 eng </controlfield>
+ <datafield tag="010" ind1=" " ind2=" ">
+ <subfield code="a"> 2010053400</subfield>
+ </datafield>
+ <datafield tag="040" ind1=" " ind2=" ">
+ <subfield code="a">DLC</subfield>
+ <subfield code="b">eng</subfield>
+ <subfield code="c">DLC</subfield>
+ <subfield code="d">RCJ</subfield>
+ <subfield code="d">CLU</subfield>
+ <subfield code="d">YDXCP</subfield>
+ <subfield code="d">BWX</subfield>
+ <subfield code="d">YLS</subfield>
+ <subfield code="d">CDX</subfield>
+ <subfield code="d">ONA</subfield>
+ <subfield code="d">OCLCF</subfield>
+ <subfield code="d">OCLCO</subfield>
+ <subfield code="d">OCL</subfield>
+ <subfield code="d">OCLCQ</subfield>
+ <subfield code="d">OCLCO</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">9780735587786</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">0735587787</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">9780735587793</subfield>
+ <subfield code="q">(pbk. ;</subfield>
+ <subfield code="q">teacher's manual)</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">0735587795</subfield>
+ <subfield code="q">(pbk. ;</subfield>
+ <subfield code="q">teacher's manual)</subfield>
+ </datafield>
+ <datafield tag="035" ind1=" " ind2=" ">
+ <subfield code="a">(OCoLC)694080497</subfield>
+ </datafield>
+ <datafield tag="042" ind1=" " ind2=" ">
+ <subfield code="a">pcc</subfield>
+ </datafield>
+ <datafield tag="043" ind1=" " ind2=" ">
+ <subfield code="a">n-us---</subfield>
+ </datafield>
+ <datafield tag="050" ind1="0" ind2="0">
+ <subfield code="a">KF9619</subfield>
+ <subfield code="b">.C62 2011</subfield>
+ </datafield>
+ <datafield tag="082" ind1="0" ind2="0">
+ <subfield code="a">345.73/05</subfield>
+ <subfield code="2">22</subfield>
+ </datafield>
+ <datafield tag="049" ind1=" " ind2=" ">
+ <subfield code="a">EQNX</subfield>
+ </datafield>
+ <datafield tag="245" ind1="0" ind2="0">
+ <subfield code="a">Comprehensive criminal procedure /</subfield>
+ <subfield code="c">Ronald Jay Allen [and others].</subfield>
+ </datafield>
+ <datafield tag="250" ind1=" " ind2=" ">
+ <subfield code="a">3rd ed.</subfield>
+ </datafield>
+ <datafield tag="260" ind1=" " ind2=" ">
+ <subfield code="a">Austin :</subfield>
+ <subfield code="b">Wolters Kluwer Law & Business,</subfield>
+ <subfield code="c">©2011.</subfield>
+ </datafield>
+ <datafield tag="300" ind1=" " ind2=" ">
+ <subfield code="a">xxxvi, 1699 pages ;</subfield>
+ <subfield code="c">26 cm +</subfield>
+ <subfield code="e">teacher's manual.</subfield>
+ </datafield>
+ <datafield tag="336" ind1=" " ind2=" ">
+ <subfield code="a">text</subfield>
+ <subfield code="b">txt</subfield>
+ <subfield code="2">rdacontent</subfield>
+ </datafield>
+ <datafield tag="337" ind1=" " ind2=" ">
+ <subfield code="a">unmediated</subfield>
+ <subfield code="b">n</subfield>
+ <subfield code="2">rdamedia</subfield>
+ </datafield>
+ <datafield tag="338" ind1=" " ind2=" ">
+ <subfield code="a">volume</subfield>
+ <subfield code="b">nc</subfield>
+ <subfield code="2">rdacarrier</subfield>
+ </datafield>
+ <datafield tag="490" ind1="1" ind2=" ">
+ <subfield code="a">Aspen casebook series</subfield>
+ </datafield>
+ <datafield tag="504" ind1=" " ind2=" ">
+ <subfield code="a">Includes bibliographical references and index.</subfield>
+ </datafield>
+ <datafield tag="505" ind1="0" ind2=" ">
+ <subfield code="a">Introduction to the criminal justice "system" -- The idea of due process -- The right to counsel and other assistance -- The rise and fall of Boyd v. United States -- The Fourth Amendment -- The Fifth Amendment -- Complex investigations in the Fourth Amendment's shadow -- Bail and pretrial detention -- The charging decision -- Pretrial screening and the grand jury -- The scope of the prosecution -- Discovery and disclosure -- Guilty pleas and plea bargaining -- The jury and the criminal trial -- Sentencing -- Double jeopardy -- Appellate and collateral review.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Criminal procedure</subfield>
+ <subfield code="z">United States.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="7">
+ <subfield code="a">Criminal procedure.</subfield>
+ <subfield code="2">fast</subfield>
+ <subfield code="0">(OCoLC)fst00883421</subfield>
+ </datafield>
+ <datafield tag="651" ind1=" " ind2="7">
+ <subfield code="a">United States.</subfield>
+ <subfield code="2">fast</subfield>
+ <subfield code="0">(OCoLC)fst01204155</subfield>
+ </datafield>
+ <datafield tag="655" ind1=" " ind2="7">
+ <subfield code="a">Casebooks.</subfield>
+ <subfield code="2">fast</subfield>
+ <subfield code="0">(OCoLC)fst01774238</subfield>
+ </datafield>
+ <datafield tag="655" ind1=" " ind2="7">
+ <subfield code="a">Casebooks.</subfield>
+ <subfield code="2">lcgft</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2=" ">
+ <subfield code="a">Allen, Ronald J.</subfield>
+ <subfield code="q">(Ronald Jay),</subfield>
+ <subfield code="d">1948-</subfield>
+ </datafield>
+ <datafield tag="830" ind1=" " ind2="0">
+ <subfield code="a">Aspen casebook series.</subfield>
+ </datafield>
+</record>$record$);
+
+SELECT cmp_ok(
+ (SELECT attrs FROM metabib.record_attr WHERE id=(SELECT CURRVAL('biblio.record_entry_id_seq'))),
+ '@>',
+ $attr$
+ "conf"=>"0",
+ "cont"=>"b",
+ "ctry"=>"txu",
+ "fest"=>"0",
+ "ills"=>" ",
+ "indx"=>"1",
+ "cont1"=>"b",
+ "date1"=>"2011",
+ "date2"=>"9999",
+ "ills1"=>" ",
+ "audience"=>" ",
+ "cat_form"=>"a",
+ "language"=>"eng",
+ "lit_form"=>"0",
+ "bib_level"=>"m",
+ "item_lang"=>"eng",
+ "item_type"=>"a",
+ "media_type"=>"unmediated",
+ "pub_status"=>"s",
+ "icon_format"=>"book",
+ "carrier_type"=>"volume",
+ "content_type"=>"text",
+ "search_format"=>"book",
+ "mr_hold_format"=>"book"
+ $attr$,
+ 'all expected record attributes found'
+);
+
+SELECT cmp_ok(
+ (SELECT attrs FROM metabib.record_attr WHERE id=(SELECT CURRVAL('biblio.record_entry_id_seq'))),
+ '<@',
+ $attr$
+ "conf"=>"0",
+ "cont"=>"b",
+ "ctry"=>"txu",
+ "fest"=>"0",
+ "ills"=>" ",
+ "indx"=>"1",
+ "cont1"=>"b",
+ "date1"=>"2011",
+ "date2"=>"9999",
+ "ills1"=>" ",
+ "audience"=>" ",
+ "cat_form"=>"a",
+ "language"=>"eng",
+ "lit_form"=>"0",
+ "bib_level"=>"m",
+ "item_lang"=>"eng",
+ "item_type"=>"a",
+ "media_type"=>"unmediated",
+ "pub_status"=>"s",
+ "icon_format"=>"book",
+ "carrier_type"=>"volume",
+ "content_type"=>"text",
+ "search_format"=>"book",
+ "mr_hold_format"=>"book"
+ $attr$,
+ 'no extra record attributes found'
+);
+
+ROLLBACK;
commit 6cc1d1e0911f4dd4a92d17a9b60025888b080bff
Author: Galen Charlton <gmc at esilibrary.com>
Date: Fri Jul 29 11:24:07 2016 -0400
LP#1588543: further speed up record attribute ingest
Following a suggestion from Dan Wells, this patch adds
an index on config.coded_value_map(ctype). My testing shows
that the index by itself can halve the time required to
fully ingest a new record; in conjunction with the
change in the previous patch, full reingest time is
reduced by about 60%.
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 0a00b31..4e37f2c 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -821,6 +821,8 @@ CREATE TABLE config.coded_value_map (
-- coded value represents
);
+CREATE INDEX config_coded_value_map_ctype_idx ON config.coded_value_map (ctype);
+
CREATE VIEW config.language_map AS SELECT code, value FROM config.coded_value_map WHERE ctype = 'item_lang';
CREATE VIEW config.bib_level_map AS SELECT code, value FROM config.coded_value_map WHERE ctype = 'bib_level';
CREATE VIEW config.item_form_map AS SELECT code, value FROM config.coded_value_map WHERE ctype = 'item_form';
commit cf3bd149e53585676a517d38093f2470dbcfaaeb
Author: Dan Wells <dbw2 at calvin.edu>
Date: Wed Jun 1 14:36:29 2016 -0400
LP#1588543: Speed up record attribute ingest
A huge number of rows (700+) have been added to
config.record_attr_definition in recent versions of Evergreen. However,
these rows do not actually define any indexing at all, but are used for
other purposes. The reingest process currently churns over all of these
rows to ultimately do nothing.
This commit limits the group of attributes considered at ingest to only
those which might index something. Testing indicates a substantial
speed boost to typical use of metabib.reingest_record_attributes()
(over 2x speedup).
Signed-off-by: Dan Wells <dbw2 at calvin.edu>
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index 0d3fb00..1c922fa 100644
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -1346,7 +1346,16 @@ DECLARE
BEGIN
IF attr_list IS NULL OR rdeleted THEN -- need to do the full dance on INSERT or undelete
- SELECT ARRAY_AGG(name) INTO attr_list FROM config.record_attr_definition;
+ SELECT ARRAY_AGG(name) INTO attr_list FROM config.record_attr_definition
+ WHERE (
+ tag IS NOT NULL OR
+ fixed_field IS NOT NULL OR
+ xpath IS NOT NULL OR
+ phys_char_sf IS NOT NULL OR
+ composite
+ ) AND (
+ filter OR sorter
+ );
END IF;
IF rmarc IS NULL THEN
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.schema.config.sql | 4 +-
Open-ILS/src/sql/Pg/030.schema.metabib.sql | 11 +-
.../sql/Pg/t/lp1588543_marc_record_attributes.pg | 211 ++++++++++++++++++++
...ql => 0985.schema.speed_record_attr_ingest.sql} | 24 ++-
4 files changed, 241 insertions(+), 9 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/t/lp1588543_marc_record_attributes.pg
copy Open-ILS/src/sql/Pg/upgrade/{0936.schema.fix_ingest_of_xpath_based_attrs.sql => 0985.schema.speed_record_attr_ingest.sql} (93%)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list