[open-ils-commits] [GIT] Evergreen ILS branch rel_2_1 updated. 6f599fa7378cb4c59d81d927b96d42235ab81fcb
Evergreen Git
git at git.evergreen-ils.org
Thu Jul 28 12:56:52 EDT 2011
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, rel_2_1 has been updated
via 6f599fa7378cb4c59d81d927b96d42235ab81fcb (commit)
via 970eaf77c399bff37f996cf2b6cb65a2e27fd5a7 (commit)
via d0c76061f617277abeefd48e0e2dd5bf43dcf185 (commit)
via 186f2d1a8ba17be65a4a26258dc8cba97d8c87af (commit)
from 1a76ff0d1cc0f60e0a494e537e6d6474d3902fc2 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 6f599fa7378cb4c59d81d927b96d42235ab81fcb
Author: Bill Erickson <berick at esilibrary.com>
Date: Thu Jul 28 12:34:28 2011 -0400
Stamped upgrade for user/dbs/oclcnumbers
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index bbc4475..9b11ed6 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -57,7 +57,7 @@ CREATE TABLE config.upgrade_log (
install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
-INSERT INTO config.upgrade_log (version) VALUES ('0583'); -- miker/gmcharlt
+INSERT INTO config.upgrade_log (version) VALUES ('0587'); -- dbs/berick
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
similarity index 98%
rename from Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
rename to Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
index 784744d..ce7638c 100644
--- a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0587.maintain_control_numbers_with_OCLC_awareness.sql
@@ -1,5 +1,7 @@
BEGIN;
+INSERT INTO config.upgrade_log (version) VALUES ('0587'); -- dbs/berick
+
CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$
use strict;
use MARC::Record;
commit 970eaf77c399bff37f996cf2b6cb65a2e27fd5a7
Author: Dan Wells <dbw2 at calvin.edu>
Date: Thu Jul 21 12:14:02 2011 -0400
Prevent control numbers from duplicating
In maintain_control_numbers(), if you have more than one 001 or
003, and at least one is the correct one, you will end up
duplicating the correct number every time the record is saved.
For example, if the incoming record was record ID # 12345, and
the record contained:
001: 12345
001: ocm10202
... then after maintain_control_numbers() ran, the record would
contain:
001: 12345
001: 12345
Also, clean up some leftovers from the refactor I suggested in the
previous commit.
Signed-off-by: Dan Wells <dbw2 at calvin.edu>
Signed-off-by: Dan Scott <dscott at laurentian.ca>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index fefae05..353c7f2 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -553,9 +553,7 @@ foreach my $id_field ('001', '003') {
} else {
# Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
foreach my $control (@controls) {
- unless ($control->data() eq $spec_value) {
- $record->delete_field($control);
- }
+ $record->delete_field($control);
}
$record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
$create = 1;
@@ -581,9 +579,6 @@ if ($munge and not $create) {
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
-
- # Update the list of SCNs to avoid duplicates
- @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
index dd386f0..784744d 100644
--- a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
@@ -66,9 +66,7 @@ foreach my $id_field ('001', '003') {
} else {
# Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
foreach my $control (@controls) {
- unless ($control->data() eq $spec_value) {
- $record->delete_field($control);
- }
+ $record->delete_field($control);
}
$record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
$create = 1;
@@ -94,9 +92,6 @@ if ($munge and not $create) {
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
-
- # Update the list of SCNs to avoid duplicates
- @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
commit d0c76061f617277abeefd48e0e2dd5bf43dcf185
Author: Dan Scott <dan at coffeecode.net>
Date: Thu Jul 21 11:12:36 2011 -0400
Add an unwrapped upgrade script for OCLC-aware control numbers
The upgrade script creates the SCN and LCCN search indexes and
populates them using metabib.full_rec as a source; cheating, but
the normalized values will work for non-exact search purposes. Sites can
reingest all of their bibs if they want exact search to work.
Also, refactor maintain_control_numbers() using a suggestion by
Dan Wells to eliminate some duplicate code.
Signed-off-by: Dan Scott <dscott at laurentian.ca>
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index 2272929..fefae05 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -567,6 +567,7 @@ my $cn = $record->field('001')->data();
if ($cn =~ /^oc[nm]/) {
$cn =~ s/^oc[nm]0*(\d+)/$1/;
$record->field('003')->data('OCoLC');
+ $create = 0;
}
# Now, if we need to munge the 001, we will first push the existing 001/003
@@ -587,13 +588,6 @@ if ($munge and not $create) {
# Set the 001/003 and update the MARC
if ($create or $munge) {
- my $scn = "(" . $record->field('003')->data() . ")" . $cn;
-
- # Do not create duplicate 035 fields
- unless (grep $_->subfield('a') eq $scn, @scns) {
- $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
- }
-
$record->field('001')->data($rec_id);
$record->field('003')->data($ou_cni);
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index b8915dc..6eda704 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -97,7 +97,9 @@ INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath )
(27, 'identifier', 'bibid', oils_i18n_gettext(27, 'Internal ID', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='901']/marc:subfield[@code='c']$$ );
-- 28 == authority_id in master
INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
- (29, 'identifier', 'system_control_number', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+ (29, 'identifier', 'scn', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (30, 'identifier', 'lccn', oils_i18n_gettext(28, 'LC Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='010']/marc:subfield[@code="a" or @code='z']$$);
SELECT SETVAL('config.metabib_field_id_seq'::TEXT, (SELECT MAX(id) FROM config.metabib_field), TRUE);
diff --git a/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
new file mode 100644
index 0000000..dd386f0
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/xxxx.maintain_control_numbers_with_OCLC_awareness.sql
@@ -0,0 +1,148 @@
+BEGIN;
+
+CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$
+use strict;
+use MARC::Record;
+use MARC::File::XML (BinaryEncoding => 'UTF-8');
+use MARC::Charset;
+use Encode;
+use Unicode::Normalize;
+
+MARC::Charset->assume_unicode(1);
+
+my $record = MARC::Record->new_from_xml($_TD->{new}{marc});
+my $schema = $_TD->{table_schema};
+my $rec_id = $_TD->{new}{id};
+
+# Short-circuit if maintaining control numbers per MARC21 spec is not enabled
+my $enable = spi_exec_query("SELECT enabled FROM config.global_flag WHERE name = 'cat.maintain_control_numbers'");
+if (!($enable->{processed}) or $enable->{rows}[0]->{enabled} eq 'f') {
+ return;
+}
+
+# Get the control number identifier from an OU setting based on $_TD->{new}{owner}
+my $ou_cni = 'EVRGRN';
+
+my $owner;
+if ($schema eq 'serial') {
+ $owner = $_TD->{new}{owning_lib};
+} else {
+ # are.owner and bre.owner can be null, so fall back to the consortial setting
+ $owner = $_TD->{new}{owner} || 1;
+}
+
+my $ous_rv = spi_exec_query("SELECT value FROM actor.org_unit_ancestor_setting('cat.marc_control_number_identifier', $owner)");
+if ($ous_rv->{processed}) {
+ $ou_cni = $ous_rv->{rows}[0]->{value};
+ $ou_cni =~ s/"//g; # Stupid VIM syntax highlighting"
+} else {
+ # Fall back to the shortname of the OU if there was no OU setting
+ $ous_rv = spi_exec_query("SELECT shortname FROM actor.org_unit WHERE id = $owner");
+ if ($ous_rv->{processed}) {
+ $ou_cni = $ous_rv->{rows}[0]->{shortname};
+ }
+}
+
+my ($create, $munge) = (0, 0);
+
+my @scns = $record->field('035');
+
+foreach my $id_field ('001', '003') {
+ my $spec_value;
+ my @controls = $record->field($id_field);
+
+ if ($id_field eq '001') {
+ $spec_value = $rec_id;
+ } else {
+ $spec_value = $ou_cni;
+ }
+
+ # Create the 001/003 if none exist
+ if (scalar(@controls) == 1) {
+ # Only one field; check to see if we need to munge it
+ unless (grep $_->data() eq $spec_value, @controls) {
+ $munge = 1;
+ }
+ } else {
+ # Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match
+ foreach my $control (@controls) {
+ unless ($control->data() eq $spec_value) {
+ $record->delete_field($control);
+ }
+ }
+ $record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value));
+ $create = 1;
+ }
+}
+
+my $cn = $record->field('001')->data();
+# Special handling of OCLC numbers, often found in records that lack 003
+if ($cn =~ /^oc[nm]/) {
+ $cn =~ s/^oc[nm]0*(\d+)/$1/;
+ $record->field('003')->data('OCoLC');
+ $create = 0;
+}
+
+# Now, if we need to munge the 001, we will first push the existing 001/003
+# into the 035; but if the record did not have one (and one only) 001 and 003
+# to begin with, skip this process
+if ($munge and not $create) {
+
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
+
+ # Do not create duplicate 035 fields
+ unless (grep $_->subfield('a') eq $scn, @scns) {
+ $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
+ }
+
+ # Update the list of SCNs to avoid duplicates
+ @scns = $record->field('035');
+}
+
+# Set the 001/003 and update the MARC
+if ($create or $munge) {
+ $record->field('001')->data($rec_id);
+ $record->field('003')->data($ou_cni);
+
+ my $xml = $record->as_xml_record();
+ $xml =~ s/\n//sgo;
+ $xml =~ s/^<\?xml.+\?\s*>//go;
+ $xml =~ s/>\s+</></go;
+ $xml =~ s/\p{Cc}//go;
+
+ # Embed a version of OpenILS::Application::AppUtils->entityize()
+ # to avoid having to set PERL5LIB for PostgreSQL as well
+
+ # If we are going to convert non-ASCII characters to XML entities,
+ # we had better be dealing with a UTF8 string to begin with
+ $xml = decode_utf8($xml);
+
+ $xml = NFC($xml);
+
+ # Convert raw ampersands to entities
+ $xml =~ s/&(?!\S+;)/&/gso;
+
+ # Convert Unicode characters to entities
+ $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
+
+ $xml =~ s/[\x00-\x1f]//go;
+ $_TD->{new}{marc} = $xml;
+
+ return "MODIFY";
+}
+
+return;
+$func$ LANGUAGE PLPERLU;
+
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (29, 'identifier', 'scn', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (30, 'identifier', 'lccn', oils_i18n_gettext(28, 'LC Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='010']/marc:subfield[@code="a" or @code='z']$$);
+
+-- Far from perfect, but much faster than reingesting every record
+INSERT INTO metabib.identifier_field_entry(source, field, value)
+ SELECT record, 29, value FROM metabib.full_rec WHERE tag = '035' AND subfield = 'a';
+INSERT INTO metabib.identifier_field_entry(source, field, value)
+ SELECT record, 30, value FROM metabib.full_rec WHERE tag = '010' AND subfield IN ('a', 'z');
+
+COMMIT;
commit 186f2d1a8ba17be65a4a26258dc8cba97d8c87af
Author: Dan Scott <dan at coffeecode.net>
Date: Thu Jul 21 10:18:55 2011 -0400
Treat OCLC numbers specially in maintain_control_numbers
For some reason, many records that come with OCLC numbers in the 001
field are not populated with a 003 field. This resulted in the OCLC
number being thrown away entirely, rather than being moved into the 035,
as there was no corresponding control number identifier field.
However, given that we know that control numbers starting with "ocm" or
"ocn" are special OCLC numbers, we can generate the 035 accordingly even
in the absence of a 003. This update to maintain_control_numbers()
teaches it to do the right thing for OCLC numbers.
In addition, add an identifier search index so we can easily find
records by their system control numbers (035 $a subfields).
Signed-off-by: Dan Scott <dscott at laurentian.ca>
Signed-off-by: Bill Erickson <berick at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql
index 4574c83..2272929 100644
--- a/Open-ILS/src/sql/Pg/002.functions.config.sql
+++ b/Open-ILS/src/sql/Pg/002.functions.config.sql
@@ -562,20 +562,38 @@ foreach my $id_field ('001', '003') {
}
}
+my $cn = $record->field('001')->data();
+# Special handling of OCLC numbers, often found in records that lack 003
+if ($cn =~ /^oc[nm]/) {
+ $cn =~ s/^oc[nm]0*(\d+)/$1/;
+ $record->field('003')->data('OCoLC');
+}
+
# Now, if we need to munge the 001, we will first push the existing 001/003
# into the 035; but if the record did not have one (and one only) 001 and 003
# to begin with, skip this process
if ($munge and not $create) {
- my $scn = "(" . $record->field('003')->data() . ")" . $record->field('001')->data();
+
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
# Do not create duplicate 035 fields
unless (grep $_->subfield('a') eq $scn, @scns) {
$record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
}
+
+ # Update the list of SCNs to avoid duplicates
+ @scns = $record->field('035');
}
# Set the 001/003 and update the MARC
if ($create or $munge) {
+ my $scn = "(" . $record->field('003')->data() . ")" . $cn;
+
+ # Do not create duplicate 035 fields
+ unless (grep $_->subfield('a') eq $scn, @scns) {
+ $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn));
+ }
+
$record->field('001')->data($rec_id);
$record->field('003')->data($ou_cni);
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 9d85859..b8915dc 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -95,6 +95,9 @@ INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath )
(26, 'identifier', 'tcn', oils_i18n_gettext(26, 'Title Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='901']/marc:subfield[@code='a']$$ );
INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath ) VALUES
(27, 'identifier', 'bibid', oils_i18n_gettext(27, 'Internal ID', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='901']/marc:subfield[@code='c']$$ );
+-- 28 == authority_id in master
+INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath) VALUES
+ (29, 'identifier', 'system_control_number', oils_i18n_gettext(28, 'System Control Number', 'cmf', 'label'), 'marcxml', $$//marc:datafield[@tag='035']/marc:subfield[@code="a"]$$);
SELECT SETVAL('config.metabib_field_id_seq'::TEXT, (SELECT MAX(id) FROM config.metabib_field), TRUE);
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.functions.config.sql | 15 +++++++--
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 5 +++
...intain_control_numbers_with_OCLC_awareness.sql} | 31 ++++++++++++++++---
4 files changed, 43 insertions(+), 10 deletions(-)
copy Open-ILS/src/sql/Pg/upgrade/{0465.function.maintain_control_numbers.sql => 0587.maintain_control_numbers_with_OCLC_awareness.sql} (72%)
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list