[open-ils-commits] [GIT] Evergreen ILS branch master updated. ab28c7b8def4b015b92bf954dec5bbb5dd8143d9
Evergreen Git
git at git.evergreen-ils.org
Thu Jul 28 12:38:02 EDT 2011
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via ab28c7b8def4b015b92bf954dec5bbb5dd8143d9 (commit)
via 6759a889bd1c3beb02255cd68ca5cf41702b0822 (commit)
via 871fab76de691ad1bcc0331c1f27b3b4d3634a47 (commit)
via a9e646657cf4693405af679a95cdea0916d874e2 (commit)
from f88ac0f74d89ed215e3d1d93aca9ccbd284ede49 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit ab28c7b8def4b015b92bf954dec5bbb5dd8143d9
Author: Bill Erickson <berick at esilibrary.com>
Date: Thu Jul 28 12:34:28 2011 -0400
Stamped upgrade for user/dbs/oclcnumbers
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index c1849b9..2ab29f9 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -86,7 +86,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0586', :eg_version); -- senator/berick
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0587', :eg_version); -- dbs/berick
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
similarity index 98%
rename from Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
rename to Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
index 784744d..a583250 100644
--- a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
@@ -1,5 +1,7 @@
BEGIN;
+SELECT evergreen.upgrade_deps_block_check('0587', :eg_version);
+
CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$
use strict;
use MARC::Record;
commit 6759a889bd1c3beb02255cd68ca5cf41702b0822
Author: Dan Wells <dbw2 at calvin.edu>
Date: Thu Jul 21 12:14:02 2011 -0400
Prevent control numbers from duplicating
In maintain_control_numbers(), if you have more than one 001 or
003, and at least one is the correct one, you will end up
duplicating the correct number every time the record is saved.
For example, if the incoming record was record ID # 12345, and
the record contained:
001: 12345
001: ocm10202
... then after maintain_control_numbers() ran, the record would
contain:
001: 12345
001: 12345
Also, clean up some leftovers from the refactor I suggested in the
previous commit.
Signed-off-by: Dan Wells <dbw2 at calvin.edu>
Signed-off-by: Dan Scott <dscott at laurentian.ca>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index fefae05..353c7f2 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -553,9 +553,7 @@ foreach my $id_field ('001', '003') {
} else {
# Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
foreach my $control (@controls) {
- unless ($control->data() eq $spec_value) {
- $record->delete_field($control);
- }
+ $record->delete_field($control);
}
$record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
$create = 1;
@@ -581,9 +579,6 @@ if ($munge and not $create) {
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
-
- # Update the list of SCNs to avoid duplicates
- @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
index dd386f0..784744d 100644
--- a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
@@ -66,9 +66,7 @@ foreach my $id_field ('001', '003') {
} else {
# Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
foreach my $control (@controls) {
- unless ($control->data() eq $spec_value) {
- $record->delete_field($control);
- }
+ $record->delete_field($control);
}
$record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
$create = 1;
@@ -94,9 +92,6 @@ if ($munge and not $create) {
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
-
- # Update the list of SCNs to avoid duplicates
- @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
commit 871fab76de691ad1bcc0331c1f27b3b4d3634a47
Author: Dan Scott <dan at coffeecode.net>
Date: Thu Jul 21 11:12:36 2011 -0400
Add an unwrapped upgrade script for OCLC-aware control numbers
The upgrade script creates the SCN and LCCN search indexes and
populates them using metabib.full_rec as a source; cheating, but
the normalized values will work for non-exact search purposes. Sites can
reingest all of their bibs if they want exact search to work.
Also, refactor maintain_control_numbers() using a suggestion by
Dan Wells to eliminate some duplicate code.
Signed-off-by: Dan Scott <dscott at laurentian.ca>
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index 2272929..fefae05 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -567,6 +567,7 @@ my $cn = $record->field('001')->data();
if ($cn =~ /^oc[nm]/) {
$cn =~ s/^oc[nm]0*(\d+)/$1/;
$record->field('003')->data('OCoLC');
+ $create = 0;
}
# Now, if we need to munge the 001, we will first push the existing 001/003
@@ -587,13 +588,6 @@ if ($munge and not $create) {
# Set the 001/003 and update the MARC
if ($create or $munge) {
- my $scn = "(" . $record->field('003')->data() . ")" . $cn;
-
- # Do not create duplicate 035 fields
- unless (grep $_->subfield('a') eq $scn, @scns) {
- $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
- }
-
$record->field('001')->data($rec_id);
$record->field('003')->data($ou_cni);
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 8a88cad..fd4c6c4 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -98,7 +98,9 @@ INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath )
INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, search_field, facet_field) VALUES
(28, 'identifier', 'authority_id', oils_i18n_gettext(28, 'Authority Record ID', 'cmf', 'label'), 'marcxml', '//marc:datafield/marc:subfield[@code="0"]', FALSE, TRUE);
INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
- (29, 'identifier', 'system_control_number', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+ (29, 'identifier', 'scn', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (30, 'identifier', 'lccn', oils_i18n_gettext(28, 'LC Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='010']/marc:subfield[@code="a" or @code='z']$$);
SELECT SETVAL('config.metabib_field_id_seq'::TEXT, (SELECT MAX(id) FROM config.metabib_field), TRUE);
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
new file mode 100644
index 0000000..dd386f0
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
@@ -0,0 +1,148 @@
+BEGIN;
+
+CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$
+use strict;
+use MARC::Record;
+use MARC::File::XML (BinaryEncoding => 'UTF-8');
+use MARC::Charset;
+use Encode;
+use Unicode::Normalize;
+
+MARC::Charset->assume_unicode(1);
+
+my $record = MARC::Record->new_from_xml($_TD->{new}{marc});
+my $schema = $_TD->{table_schema};
+my $rec_id = $_TD->{new}{id};
+
+# Short-circuit if maintaining control numbers per MARC21 spec is not enabled
+my $enable = spi_exec_query("SELECT enabled FROM config.global_flag WHERE name = 'cat.maintain_control_numbers'");
+if (!($enable->{processed}) or $enable->{rows}[0]->{enabled} eq 'f') {
+ return;
+}
+
+# Get the control number identifier from an OU setting based on $_TD->{new}{owner}
+my $ou_cni = 'EVRGRN';
+
+my $owner;
+if ($schema eq 'serial') {
+ $owner = $_TD->{new}{owning_lib};
+} else {
+ # are.owner and bre.owner can be null, so fall back to the consortial setting
+ $owner = $_TD->{new}{owner} || 1;
+}
+
+my $ous_rv = spi_exec_query("SELECT value FROM actor.org_unit_ancestor_setting('cat.marc_control_number_identifier', $owner)");
+if ($ous_rv->{processed}) {
+ $ou_cni = $ous_rv->{rows}[0]->{value};
+ $ou_cni =~ s/"//g; # Stupid VIM syntax highlighting"
+} else {
+ # Fall back to the shortname of the OU if there was no OU setting
+ $ous_rv = spi_exec_query("SELECT shortname FROM actor.org_unit WHERE id = $owner");
+ if ($ous_rv->{processed}) {
+ $ou_cni = $ous_rv->{rows}[0]->{shortname};
+ }
+}
+
+my ($create, $munge) = (0, 0);
+
+my @scns = $record->field('035');
+
+foreach my $id_field ('001', '003') {
+ my $spec_value;
+ my @controls = $record->field($id_field);
+
+ if ($id_field eq '001') {
+ $spec_value = $rec_id;
+ } else {
+ $spec_value = $ou_cni;
+ }
+
+ # Create the 001/003 if none exist
+ if (scalar(@controls) == 1) {
+ # Only one field; check to see if we need to munge it
+ unless (grep $_->data() eq $spec_value, @controls) {
+ $munge = 1;
+ }
+ } else {
+ # Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
+ foreach my $control (@controls) {
+ unless ($control->data() eq $spec_value) {
+ $record->delete_field($control);
+ }
+ }
+ $record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
+ $create = 1;
+ }
+}
+
+my $cn = $record->field('001')->data();
+# Special handling of OCLC numbers, often found in records that lack 003
+if ($cn =~ /^oc[nm]/) {
+ $cn =~ s/^oc[nm]0*(\d+)/$1/;
+ $record->field('003')->data('OCoLC');
+ $create = 0;
+}
+
+# Now, if we need to munge the 001, we will first push the existing 001/003
+# into the 035; but if the record did not have one (and one only) 001 and 003
+# to begin with, skip this process
+if ($munge and not $create) {
+
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
+
+ # Do not create duplicate 035 fields
+ unless (grep $_->subfield('a') eq $scn, @scns) {
+ $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
+ }
+
+ # Update the list of SCNs to avoid duplicates
+ @scns = $record->field('035');
+}
+
+# Set the 001/003 and update the MARC
+if ($create or $munge) {
+ $record->field('001')->data($rec_id);
+ $record->field('003')->data($ou_cni);
+
+ my $xml = $record->as_xml_record();
+ $xml =~ s/\n//sgo;
+ $xml =~ s/^<\?xml.+\?\s*>//go;
+ $xml =~ s/>\s+</></go;
+ $xml =~ s/\p{Cc}//go;
+
+ # Embed a version of OpenILS::Application::AppUtils->entityize()
+ # to avoid having to set PERL5LIB for PostgreSQL as well
+
+ # If we are going to convert non-ASCII characters to XML entities,
+ # we had better be dealing with a UTF8 string to begin with
+ $xml = decode_utf8($xml);
+
+ $xml = NFC($xml);
+
+ # Convert raw ampersands to entities
+ $xml =~ s/&(?!\S+;)/&/gso;
+
+ # Convert Unicode characters to entities
+ $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
+
+ $xml =~ s/[\x00-\x1f]//go;
+ $_TD->{new}{marc} = $xml;
+
+ return "MODIFY";
+}
+
+return;
+$func$ LANGUAGE PLPERLU;
+
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (29, 'identifier', 'scn', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (30, 'identifier', 'lccn', oils_i18n_gettext(28, 'LC Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='010']/marc:subfield[@code="a" or @code='z']$$);
+
+-- Far from perfect, but much faster than reingesting every record
+INSERT INTO metabib.identifier_field_entry(source, field, value)
+ SELECT record, 29, value FROM metabib.full_rec WHERE tag = '035' AND subfield = 'a';
+INSERT INTO metabib.identifier_field_entry(source, field, value)
+ SELECT record, 30, value FROM metabib.full_rec WHERE tag = '010' AND subfield IN ('a', 'z');
+
+COMMIT;
commit a9e646657cf4693405af679a95cdea0916d874e2
Author: Dan Scott <dan at coffeecode.net>
Date: Thu Jul 21 10:18:55 2011 -0400
Treat OCLC numbers specially in maintain_control_numbers
For some reason, many records that come with OCLC numbers in the 001
field are not populated with a 003 field. This resulted in the OCLC
number being thrown away entirely, rather than being moved into the 035,
as there was no corresponding control number identifier field.
However, given that we know that control numbers starting with "ocm" or
"ocn" are special OCLC numbers, we can generate the 035 accordingly even
in the absence of a 003. This update to maintain_control_numbers()
teaches it to do the right thing for OCLC numbers.
In addition, add an identifier search index so we can easily find
records by their system control numbers (035 $a subfields).
Signed-off-by: Dan Scott <dscott at laurentian.ca>
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index 4574c83..2272929 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -562,20 +562,38 @@ foreach my $id_field ('001', '003') {
}
}
+my $cn = $record->field('001')->data();
+# Special handling of OCLC numbers, often found in records that lack 003
+if ($cn =~ /^oc[nm]/) {
+ $cn =~ s/^oc[nm]0*(\d+)/$1/;
+ $record->field('003')->data('OCoLC');
+}
+
# Now, if we need to munge the 001, we will first push the existing 001/003
# into the 035; but if the record did not have one (and one only) 001 and 003
# to begin with, skip this process
if ($munge and not $create) {
- my $scn = "(" . $record->field('003')->data() . ")" . $record->field('001')->data();
+
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
# Do not create duplicate 035 fields
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
+
+ # Update the list of SCNs to avoid duplicates
+ @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
if ($create or $munge) {
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
+
+ # Do not create duplicate 035 fields
+ unless (grep $_->subfield('a') eq $scn, @scns) {
+ $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
+ }
+
$record->field('001')->data($rec_id);
$record->field('003')->data($ou_cni);
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 95f4f34..8a88cad 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -97,6 +97,8 @@ INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath )
(27, 'identifier', 'bibid', oils_i18n_gettext(27, 'Internal ID', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='901']/marc:subfield[@code='c']$$ );
INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, search_field, facet_field) VALUES
(28, 'identifier', 'authority_id', oils_i18n_gettext(28, 'Authority Record ID', 'cmf', 'label'), 'marcxml', '//marc:datafield/marc:subfield[@code="0"]', FALSE, TRUE);
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (29, 'identifier', 'system_control_number', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
SELECT SETVAL('config.metabib_field_id_seq'::TEXT, (SELECT MAX(id) FROM config.metabib_field), TRUE);
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.functions.config.sql | 15 +++++++--
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 4 ++
...intain_control_numbers_with_OCLC_awareness.sql} | 31 ++++++++++++++++---
4 files changed, 42 insertions(+), 10 deletions(-)
copy Open-ILS/src/sql/Pg/upgrade/{0465.function.maintain_control_numbers.sql => 0587.maintain_control_numbers_with_OCLC_awareness.sql} (72%)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list