[open-ils-commits] r17864 - in branches/rel_2_0/Open-ILS/src/sql/Pg: . upgrade (gmc)
svn at svn.open-ils.org
svn at svn.open-ils.org
Tue Sep 21 10:25:10 EDT 2010
Author: gmc
Date: 2010-09-21 10:25:07 -0400 (Tue, 21 Sep 2010)
New Revision: 17864
Added:
branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0410.schema.allow_parallel_reingest.sql
branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
Modified:
branches/rel_2_0/Open-ILS/src/sql/Pg/002.schema.config.sql
branches/rel_2_0/Open-ILS/src/sql/Pg/030.schema.metabib.sql
Log:
backport r17862 and r17863 from trunk
Two new internal_flags to improve parallel bib ingest.
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Modified: branches/rel_2_0/Open-ILS/src/sql/Pg/002.schema.config.sql
===================================================================
--- branches/rel_2_0/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-09-21 14:19:31 UTC (rev 17863)
+++ branches/rel_2_0/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-09-21 14:25:07 UTC (rev 17864)
@@ -52,11 +52,13 @@
enabled BOOL NOT NULL DEFAULT FALSE
);
INSERT INTO config.internal_flag (name) VALUES ('ingest.metarecord_mapping.skip_on_insert');
+INSERT INTO config.internal_flag (name) VALUES ('ingest.metarecord_mapping.skip_on_update');
INSERT INTO config.internal_flag (name) VALUES ('ingest.reingest.force_on_same_marc');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_located_uri');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_full_rec');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_rec_descriptor');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_field_entry');
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
CREATE TABLE config.global_flag (
label TEXT NOT NULL
@@ -68,7 +70,7 @@
install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
-INSERT INTO config.upgrade_log (version) VALUES ('0409'); -- senator
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
Modified: branches/rel_2_0/Open-ILS/src/sql/Pg/030.schema.metabib.sql
===================================================================
--- branches/rel_2_0/Open-ILS/src/sql/Pg/030.schema.metabib.sql 2010-09-21 14:19:31 UTC (rev 17863)
+++ branches/rel_2_0/Open-ILS/src/sql/Pg/030.schema.metabib.sql 2010-09-21 14:25:07 UTC (rev 17864)
@@ -707,7 +707,10 @@
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
BEGIN
- DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ END IF;
INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
SELECT bib_id,
biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
@@ -735,7 +738,10 @@
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
BEGIN
- DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ END IF;
INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
@@ -748,13 +754,15 @@
fclass RECORD;
ind_data metabib.field_entry_template%ROWTYPE;
BEGIN
- FOR fclass IN SELECT * FROM config.metabib_class LOOP
- -- RAISE NOTICE 'Emptying out %', fclass.name;
- EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
- END LOOP;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
- DELETE FROM metabib.facet_entry WHERE source = bib_id;
-
FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
IF ind_data.field < 0 THEN
ind_data.field = -1 * ind_data.field;
@@ -964,7 +972,10 @@
PERFORM metabib.remap_metarecord_for_bib( NEW.id, NEW.fingerprint );
END IF;
ELSE -- we're doing an update, and we're not deleted, remap
- PERFORM metabib.remap_metarecord_for_bib( NEW.id, NEW.fingerprint );
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.metarecord_mapping.skip_on_update' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM metabib.remap_metarecord_for_bib( NEW.id, NEW.fingerprint );
+ END IF;
END IF;
RETURN NEW;
Copied: branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0410.schema.allow_parallel_reingest.sql (from rev 17862, trunk/Open-ILS/src/sql/Pg/upgrade/0410.schema.allow_parallel_reingest.sql)
===================================================================
--- branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0410.schema.allow_parallel_reingest.sql (rev 0)
+++ branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0410.schema.allow_parallel_reingest.sql 2010-09-21 14:25:07 UTC (rev 17864)
@@ -0,0 +1,74 @@
+BEGIN;
+
+INSERT INTO config.upgrade_log (version) VALUES ('0410'); -- gmc
+
+INSERT INTO config.internal_flag (name) VALUES ('ingest.metarecord_mapping.skip_on_update');
+
+-- AFTER UPDATE OR INSERT trigger for biblio.record_entry
+CREATE OR REPLACE FUNCTION biblio.indexing_ingest_or_delete () RETURNS TRIGGER AS $func$
+BEGIN
+
+ IF NEW.deleted IS TRUE THEN -- If this bib is deleted
+ DELETE FROM metabib.metarecord_source_map WHERE source = NEW.id; -- Rid ourselves of the search-estimate-killing linkage
+ DELETE FROM authority.bib_linking WHERE bib = NEW.id; -- Avoid updating fields in bibs that are no longer visible
+ RETURN NEW; -- and we're done
+ END IF;
+
+ IF TG_OP = 'UPDATE' THEN -- re-ingest?
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.reingest.force_on_same_marc' AND enabled;
+
+ IF NOT FOUND AND OLD.marc = NEW.marc THEN -- don't do anything if the MARC didn't change
+ RETURN NEW;
+ END IF;
+ END IF;
+
+ -- Record authority linking
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_authority_linking' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM biblio.map_authority_linking( NEW.id, NEW.marc );
+ END IF;
+
+ -- Flatten and insert the mfr data
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_metabib_full_rec' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM metabib.reingest_metabib_full_rec(NEW.id);
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_metabib_rec_descriptor' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM metabib.reingest_metabib_rec_descriptor(NEW.id);
+ END IF;
+ END IF;
+
+ -- Gather and insert the field entry data
+ PERFORM metabib.reingest_metabib_field_entries(NEW.id);
+
+ -- Located URI magic
+ IF TG_OP = 'INSERT' THEN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_located_uri' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM biblio.extract_located_uris( NEW.id, NEW.marc, NEW.editor );
+ END IF;
+ ELSE
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_located_uri' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM biblio.extract_located_uris( NEW.id, NEW.marc, NEW.editor );
+ END IF;
+ END IF;
+
+ -- (re)map metarecord-bib linking
+ IF TG_OP = 'INSERT' THEN -- if not deleted and performing an insert, check for the flag
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.metarecord_mapping.skip_on_insert' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM metabib.remap_metarecord_for_bib( NEW.id, NEW.fingerprint );
+ END IF;
+ ELSE -- we're doing an update, and we're not deleted, remap
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.metarecord_mapping.skip_on_update' AND enabled;
+ IF NOT FOUND THEN
+ PERFORM metabib.remap_metarecord_for_bib( NEW.id, NEW.fingerprint );
+ END IF;
+ END IF;
+
+ RETURN NEW;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;
Added: branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
===================================================================
--- branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql (rev 0)
+++ branches/rel_2_0/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql 2010-09-21 14:25:07 UTC (rev 17864)
@@ -0,0 +1,86 @@
+BEGIN;
+
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
+
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ END IF;
+ INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
+ SELECT bib_id,
+ biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Form' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'BLvl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Ctrl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'ELvl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Audn' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'LitF' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'TMat' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Desc' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'DtSt' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Lang' ),
+ ( SELECT v.value
+ FROM biblio.marc21_physical_characteristics( bib_id) p
+ JOIN config.marc21_physical_characteristic_subfield_map s ON (s.id = p.subfield)
+ JOIN config.marc21_physical_characteristic_value_map v ON (v.id = p.value)
+ WHERE p.ptype = 'v' AND s.subfield = 'e' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Date1'),
+ biblio.marc21_extract_fixed_field( bib_id, 'Date2');
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ END IF;
+ INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
+ SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT ) RETURNS VOID AS $func$
+DECLARE
+ fclass RECORD;
+ ind_data metabib.field_entry_template%ROWTYPE;
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
+
+ FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+ IF ind_data.field < 0 THEN
+ ind_data.field = -1 * ind_data.field;
+ INSERT INTO metabib.facet_entry (field, source, value)
+ VALUES (ind_data.field, ind_data.source, ind_data.value);
+ ELSE
+ EXECUTE $$
+ INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
+ VALUES ($$ ||
+ quote_literal(ind_data.field) || $$, $$ ||
+ quote_literal(ind_data.source) || $$, $$ ||
+ quote_literal(ind_data.value) ||
+ $$);$$;
+ END IF;
+
+ END LOOP;
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;
More information about the open-ils-commits
mailing list