[open-ils-commits] r17863 - in trunk/Open-ILS/src/sql/Pg: . upgrade (gmc)
svn at svn.open-ils.org
svn at svn.open-ils.org
Tue Sep 21 10:19:33 EDT 2010
Author: gmc
Date: 2010-09-21 10:19:31 -0400 (Tue, 21 Sep 2010)
New Revision: 17863
Added:
trunk/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
Modified:
trunk/Open-ILS/src/sql/Pg/002.schema.config.sql
trunk/Open-ILS/src/sql/Pg/030.schema.metabib.sql
Log:
yet another bib ingest knob
When enabled, the ingest.assume_inserts_only flag disables
checking several tables for old index data to delete; skipping this
makes a small but noticeable difference when doing an initial bib
ingest with more than one process in parallel.
Note that this flag should not be used when *re*ingesting records
unless you clear the old rows beforehand.
Signed-off-by: Galen Charlton <gmc at esilibrary.com>
Modified: trunk/Open-ILS/src/sql/Pg/002.schema.config.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-09-21 04:46:52 UTC (rev 17862)
+++ trunk/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-09-21 14:19:31 UTC (rev 17863)
@@ -58,6 +58,7 @@
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_full_rec');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_rec_descriptor');
INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_field_entry');
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
CREATE TABLE config.global_flag (
label TEXT NOT NULL
@@ -69,7 +70,7 @@
install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
-INSERT INTO config.upgrade_log (version) VALUES ('0410'); -- gmc
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
Modified: trunk/Open-ILS/src/sql/Pg/030.schema.metabib.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/030.schema.metabib.sql 2010-09-21 04:46:52 UTC (rev 17862)
+++ trunk/Open-ILS/src/sql/Pg/030.schema.metabib.sql 2010-09-21 14:19:31 UTC (rev 17863)
@@ -707,7 +707,10 @@
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
BEGIN
- DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ END IF;
INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
SELECT bib_id,
biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
@@ -735,7 +738,10 @@
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
BEGIN
- DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ END IF;
INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
@@ -748,13 +754,15 @@
fclass RECORD;
ind_data metabib.field_entry_template%ROWTYPE;
BEGIN
- FOR fclass IN SELECT * FROM config.metabib_class LOOP
- -- RAISE NOTICE 'Emptying out %', fclass.name;
- EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
- END LOOP;
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
- DELETE FROM metabib.facet_entry WHERE source = bib_id;
-
FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
IF ind_data.field < 0 THEN
ind_data.field = -1 * ind_data.field;
Added: trunk/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql (rev 0)
+++ trunk/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql 2010-09-21 14:19:31 UTC (rev 17863)
@@ -0,0 +1,86 @@
+BEGIN;
+
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
+
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+ END IF;
+ INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
+ SELECT bib_id,
+ biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Form' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'BLvl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Ctrl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'ELvl' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Audn' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'LitF' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'TMat' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Desc' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'DtSt' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Lang' ),
+ ( SELECT v.value
+ FROM biblio.marc21_physical_characteristics( bib_id) p
+ JOIN config.marc21_physical_characteristic_subfield_map s ON (s.id = p.subfield)
+ JOIN config.marc21_physical_characteristic_value_map v ON (v.id = p.value)
+ WHERE p.ptype = 'v' AND s.subfield = 'e' ),
+ biblio.marc21_extract_fixed_field( bib_id, 'Date1'),
+ biblio.marc21_extract_fixed_field( bib_id, 'Date2');
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+ END IF;
+ INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
+ SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT ) RETURNS VOID AS $func$
+DECLARE
+ fclass RECORD;
+ ind_data metabib.field_entry_template%ROWTYPE;
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
+
+ FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+ IF ind_data.field < 0 THEN
+ ind_data.field = -1 * ind_data.field;
+ INSERT INTO metabib.facet_entry (field, source, value)
+ VALUES (ind_data.field, ind_data.source, ind_data.value);
+ ELSE
+ EXECUTE $$
+ INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
+ VALUES ($$ ||
+ quote_literal(ind_data.field) || $$, $$ ||
+ quote_literal(ind_data.source) || $$, $$ ||
+ quote_literal(ind_data.value) ||
+ $$);$$;
+ END IF;
+
+ END LOOP;
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;
More information about the open-ils-commits
mailing list