[open-ils-commits] [GIT] Evergreen ILS branch rel_3_2 updated. d731c233112b6ebdf8fde12f80a3b7916f608539
Evergreen Git
git at git.evergreen-ils.org
Fri Mar 8 14:00:11 EST 2019
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, rel_3_2 has been updated
via d731c233112b6ebdf8fde12f80a3b7916f608539 (commit)
via 793286531f5c2cbb7625e8c54f2f3dbade71491a (commit)
via 028750d7c2ff992a3702533925c62b9c15b93f1e (commit)
from b493ff500fab9b70379068ef2bbc5ce2e78abf68 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit d731c233112b6ebdf8fde12f80a3b7916f608539
Author: Jason Stephenson <jason at sigio.com>
Date: Fri Mar 8 13:54:35 2019 -0500
Lp 712490: Stamping UPgrade Script
Signed-off-by: Jason Stephenson <jason at sigio.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 2d37ee2382..b6f266531c 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -92,7 +92,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1154', :eg_version); -- berick/mmorgan
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1157', :eg_version); -- berick/sandbergja/Dyrcona
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql b/Open-ILS/src/sql/Pg/upgrade/1157.schema.vandelay-replace-field-order.sql
similarity index 99%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql
rename to Open-ILS/src/sql/Pg/upgrade/1157.schema.vandelay-replace-field-order.sql
index 5fb542a021..39f59a96e7 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/1157.schema.vandelay-replace-field-order.sql
@@ -1,6 +1,6 @@
BEGIN;
--- SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('1157', :eg_version);
CREATE OR REPLACE FUNCTION vandelay.replace_field
(target_xml TEXT, source_xml TEXT, field TEXT) RETURNS TEXT AS $_$
commit 793286531f5c2cbb7625e8c54f2f3dbade71491a
Author: Bill Erickson <berickxx at gmail.com>
Date: Thu Oct 19 11:42:43 2017 -0400
LP#712490 Vandelay replace/merge PGTAP tests
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Jane Sandberg <sandbej at linnbenton.edu>
Signed-off-by: Jason Stephenson <jason at sigio.com>
diff --git a/Open-ILS/src/sql/Pg/t/lp712490-vand-replace-field-merge.pg b/Open-ILS/src/sql/Pg/t/lp712490-vand-replace-field-merge.pg
new file mode 100644
index 0000000000..ca262d8e80
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/t/lp712490-vand-replace-field-merge.pg
@@ -0,0 +1,135 @@
+BEGIN;
+
+-- Plan the tests.
+SELECT plan(5);
+
+SELECT is(
+ (SELECT vandelay.replace_field($$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader> a </leader>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-</subfield>
+ </datafield>
+</record>
+$$, $$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-2013</subfield>
+ </datafield>
+</record>
+$$, '100abcdef')),
+ '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader> a </leader><datafield tag="100" ind1="1" ind2=" "><subfield code="a">Banks, Iain M</subfield><subfield code="d">1954-2013</subfield></datafield></record>',
+ 'Replacing a value for a single existing subfield'
+);
+
+SELECT is(
+ (SELECT vandelay.replace_field($$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader> a </leader>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ </datafield>
+</record>
+$$, $$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-2013</subfield>
+ </datafield>
+</record>
+$$, '100abcdef')),
+ '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader> a </leader><datafield tag="100" ind1="1" ind2=" "><subfield code="a">Banks, Iain M</subfield><subfield code="d">1954-2013</subfield></datafield></record>',
+ 'Adding a missing field'
+);
+
+SELECT is(
+ (SELECT vandelay.replace_field($$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader> a </leader>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-</subfield>
+ </datafield>
+</record>
+$$, $$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="b">HELLO</subfield>
+ <subfield code="d">1954-2013</subfield>
+ </datafield>
+</record>
+$$, '100abcdef')),
+ '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader> a </leader><datafield tag="100" ind1="1" ind2=" "><subfield code="a">Banks, Iain M</subfield><subfield code="b">HELLO</subfield><subfield code="d">1954-2013</subfield></datafield></record>',
+ 'Splicing a new field into the existing fields'
+);
+
+SELECT is(
+ (SELECT vandelay.replace_field($$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader> a </leader>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-</subfield>
+ <subfield code="z">STAY PUT</subfield>
+ </datafield>
+</record>
+$$, $$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="b">HELLO</subfield>
+ <subfield code="y">IGNORE ME</subfield>
+ <subfield code="d">1954-2013</subfield>
+ </datafield>
+</record>
+$$, '100abcdef')),
+ '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader> a </leader><datafield tag="100" ind1="1" ind2=" "><subfield code="a">Banks, Iain M</subfield><subfield code="b">HELLO</subfield><subfield code="d">1954-2013</subfield><subfield code="z">STAY PUT</subfield></datafield></record>',
+ 'Splice new field, ignore uncontrolled source field, leave uncontrolled target field in place'
+);
+
+SELECT is(
+ (SELECT vandelay.replace_field($$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader> a </leader>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ <subfield code="d">1954-</subfield>
+ </datafield>
+</record>
+$$, $$
+<record xmlns="http://www.loc.gov/MARC21/slim"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Banks, Iain M</subfield>
+ </datafield>
+</record>
+$$, '100abcdef')),
+ '<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns="http://www.loc.gov/MARC21/slim"><leader> a </leader><datafield tag="100" ind1="1" ind2=" "><subfield code="a">Banks, Iain M</subfield></datafield></record>',
+ 'Removing a subfield'
+);
+
+SELECT * FROM finish();
+ROLLBACK;
+
commit 028750d7c2ff992a3702533925c62b9c15b93f1e
Author: Bill Erickson <berickxx at gmail.com>
Date: Wed Oct 18 17:00:59 2017 -0400
LP#712490 Vandelay merge-based field replacement
Modify the vandelay field replacement logic to merge replacement data
into existing field data where possible, instead of simply deleting then
re-adding the data. The key benefit is that subfields retain their order
in affected MARC fields. As a result, for example, propagating
authority field changes to bib records will no longer result in the $0
field being pushed to the front of the field when it was previously that
last subfield.
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Jane Sandberg <sandbej at linnbenton.edu>
Signed-off-by: Jason Stephenson <jason at sigio.com>
diff --git a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
index 1f53ad17d3..18268f465a 100644
--- a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
+++ b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql
@@ -1264,35 +1264,201 @@ CREATE OR REPLACE FUNCTION vandelay.strip_field ( xml TEXT, field TEXT ) RETURNS
$_$ LANGUAGE PLPERLU;
-CREATE OR REPLACE FUNCTION vandelay.replace_field ( target_xml TEXT, source_xml TEXT, field TEXT ) RETURNS TEXT AS $_$
-DECLARE
- xml_output TEXT;
- parsed_target TEXT;
- curr_field TEXT;
-BEGIN
+CREATE OR REPLACE FUNCTION vandelay.replace_field
+ (target_xml TEXT, source_xml TEXT, field TEXT) RETURNS TEXT AS $_$
- parsed_target := vandelay.strip_field( target_xml, ''); -- this dance normalizes the format of the xml for the IF below
- xml_output := parsed_target; -- if there are no replace rules, just return the input
+ use strict;
+ use MARC::Record;
+ use MARC::Field;
+ use MARC::File::XML (BinaryEncoding => 'UTF-8');
+ use MARC::Charset;
- FOR curr_field IN SELECT UNNEST( STRING_TO_ARRAY(field, ',') ) LOOP -- naive split, but it's the same we use in the perl
+ MARC::Charset->assume_unicode(1);
- xml_output := vandelay.strip_field( parsed_target, curr_field);
+ my $target_xml = shift;
+ my $source_xml = shift;
+ my $field_spec = shift;
- IF xml_output <> parsed_target AND curr_field ~ E'~' THEN
- -- we removed something, and there was a regexp restriction in the curr_field definition, so proceed
- xml_output := vandelay.add_field( xml_output, source_xml, curr_field, 1 );
- ELSIF curr_field !~ E'~' THEN
- -- No regexp restriction, add the curr_field
- xml_output := vandelay.add_field( xml_output, source_xml, curr_field, 0 );
- END IF;
+ my $target_r = MARC::Record->new_from_xml($target_xml);
+ my $source_r = MARC::Record->new_from_xml($source_xml);
- parsed_target := xml_output; -- in prep for any following loop iterations
+ return $target_xml unless $target_r && $source_r;
- END LOOP;
+ # Extract the field_spec components into MARC tags, subfields,
+ # and regex matches. Copied wholesale from vandelay.strip_field()
- RETURN xml_output;
-END;
-$_$ LANGUAGE PLPGSQL;
+ my @field_list = split(',', $field_spec);
+ my %fields;
+ for my $f (@field_list) {
+ $f =~ s/^\s*//; $f =~ s/\s*$//;
+ if ($f =~ /^(.{3})(\w*)(?:\[([^]]*)\])?$/) {
+ my $field = $1;
+ $field =~ s/\s+//;
+ my $sf = $2;
+ $sf =~ s/\s+//;
+ my $match = $3;
+ $match =~ s/^\s*//; $match =~ s/\s*$//;
+ $fields{$field} = { sf => [ split('', $sf) ] };
+ if ($match) {
+ my ($msf,$mre) = split('~', $match);
+ if (length($msf) > 0 and length($mre) > 0) {
+ $msf =~ s/^\s*//; $msf =~ s/\s*$//;
+ $mre =~ s/^\s*//; $mre =~ s/\s*$//;
+ $fields{$field}{match} = { sf => $msf, re => qr/$mre/ };
+ }
+ }
+ }
+ }
+
+ # Returns a flat list of subfield (code, value, code, value, ...)
+ # suitable for adding to a MARC::Field.
+ sub generate_replacement_subfields {
+ my ($source_field, $target_field, @controlled_subfields) = @_;
+
+ # Performing a wholesale field replacment.
+ # Use the entire source field as-is.
+ return map {$_->[0], $_->[1]} $source_field->subfields
+ unless @controlled_subfields;
+
+ my @new_subfields;
+
+ # Iterate over all target field subfields:
+ # 1. Keep uncontrolled subfields as is.
+ # 2. Replace values for controlled subfields when a
+ # replacement value exists on the source record.
+ # 3. Delete values for controlled subfields when no
+ # replacement value exists on the source record.
+
+ for my $target_sf ($target_field->subfields) {
+ my $subfield = $target_sf->[0];
+ my $target_val = $target_sf->[1];
+
+ if (grep {$_ eq $subfield} @controlled_subfields) {
+ if (my $source_val = $source_field->subfield($subfield)) {
+ # We have a replacement value
+ push(@new_subfields, $subfield, $source_val);
+ } else {
+ # no replacement value for controlled subfield, drop it.
+ }
+ } else {
+ # Field is not controlled. Copy it over as-is.
+ push(@new_subfields, $subfield, $target_val);
+ }
+ }
+
+ # Iterate over all subfields in the source field and back-fill
+ # any values that exist only in the source field. Insert these
+ # subfields in the same relative position they exist in the
+ # source field.
+
+ my @seen_subfields;
+ for my $source_sf ($source_field->subfields) {
+ my $subfield = $source_sf->[0];
+ my $source_val = $source_sf->[1];
+ push(@seen_subfields, $subfield);
+
+ # target field already contains this subfield,
+ # so it would have been addressed above.
+ next if $target_field->subfield($subfield);
+
+ # Ignore uncontrolled subfields.
+ next unless grep {$_ eq $subfield} @controlled_subfields;
+
+ # Adding a new subfield. Find its relative position and add
+ # it to the list under construction. Work backwards from
+ # the list of already seen subfields to find the best slot.
+
+ my $done = 0;
+ for my $seen_sf (reverse(@seen_subfields)) {
+ my $idx = @new_subfields;
+ for my $new_sf (reverse(@new_subfields)) {
+ $idx--;
+ next if $idx % 2 == 1; # sf codes are in the even slots
+
+ if ($new_subfields[$idx] eq $seen_sf) {
+ splice(@new_subfields, $idx + 2, 0, $subfield, $source_val);
+ $done = 1;
+ last;
+ }
+ }
+ last if $done;
+ }
+
+ # if no slot was found, add to the end of the list.
+ push(@new_subfields, $subfield, $source_val) unless $done;
+ }
+
+ return @new_subfields;
+ }
+
+ # MARC tag loop
+ for my $f (keys %fields) {
+ my $tag_idx = -1;
+ for my $target_field ($target_r->field($f)) {
+
+ # field spec contains a regex for this field. Confirm field on
+ # target record matches the specified regex before replacing.
+ if (exists($fields{$f}{match})) {
+ next unless (grep { $_ =~ $fields{$f}{match}{re} }
+ $target_field->subfield($fields{$f}{match}{sf}));
+ }
+
+ my @new_subfields;
+ my @controlled_subfields = @{$fields{$f}{sf}};
+
+ # If the target record has multiple matching bib fields,
+ # replace them from matching fields on the source record
+ # in a predictable order to avoid replacing with them with
+ # same source field repeatedly.
+ my @source_fields = $source_r->field($f);
+ my $source_field = $source_fields[++$tag_idx];
+
+ if (!$source_field && @controlled_subfields) {
+ # When there are more target fields than source fields
+ # and we are replacing values for subfields and not
+ # performing wholesale field replacment, use the last
+ # available source field as the input for all remaining
+ # target fields.
+ $source_field = $source_fields[$#source_fields];
+ }
+
+ if (!$source_field) {
+ # No source field exists. Delete all affected target
+ # data. This is a little bit counterintuitive, but is
+ # backwards compatible with the previous version of this
+ # function which first deleted all affected data, then
+ # replaced values where possible.
+ if (@controlled_subfields) {
+ $target_field->delete_subfield($_) for @controlled_subfields;
+ } else {
+ $target_r->delete_field($target_field);
+ }
+ next;
+ }
+
+ my @new_subfields = generate_replacement_subfields(
+ $source_field, $target_field, @controlled_subfields);
+
+ # Build the replacement field from scratch.
+ my $replacement_field = MARC::Field->new(
+ $target_field->tag,
+ $target_field->indicator(1),
+ $target_field->indicator(2),
+ @new_subfields
+ );
+
+ $target_field->replace_with($replacement_field);
+ }
+ }
+
+ $target_xml = $target_r->as_xml_record;
+ $target_xml =~ s/^<\?.+?\?>$//mo;
+ $target_xml =~ s/\n//sgo;
+ $target_xml =~ s/>\s+</></sgo;
+
+ return $target_xml;
+
+$_$ LANGUAGE PLPERLU;
CREATE OR REPLACE FUNCTION vandelay.merge_record_xml ( target_xml TEXT, source_xml TEXT, add_rule TEXT, replace_preserve_rule TEXT, strip_rule TEXT ) RETURNS TEXT AS $_$
SELECT vandelay.replace_field( vandelay.add_field( vandelay.strip_field( $1, $5) , $2, $3 ), $2, $4);
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql
new file mode 100644
index 0000000000..5fb542a021
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay-replace-field-order.sql
@@ -0,0 +1,202 @@
+BEGIN;
+
+-- SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE OR REPLACE FUNCTION vandelay.replace_field
+ (target_xml TEXT, source_xml TEXT, field TEXT) RETURNS TEXT AS $_$
+
+ use strict;
+ use MARC::Record;
+ use MARC::Field;
+ use MARC::File::XML (BinaryEncoding => 'UTF-8');
+ use MARC::Charset;
+
+ MARC::Charset->assume_unicode(1);
+
+ my $target_xml = shift;
+ my $source_xml = shift;
+ my $field_spec = shift;
+
+ my $target_r = MARC::Record->new_from_xml($target_xml);
+ my $source_r = MARC::Record->new_from_xml($source_xml);
+
+ return $target_xml unless $target_r && $source_r;
+
+ # Extract the field_spec components into MARC tags, subfields,
+ # and regex matches. Copied wholesale from vandelay.strip_field()
+
+ my @field_list = split(',', $field_spec);
+ my %fields;
+ for my $f (@field_list) {
+ $f =~ s/^\s*//; $f =~ s/\s*$//;
+ if ($f =~ /^(.{3})(\w*)(?:\[([^]]*)\])?$/) {
+ my $field = $1;
+ $field =~ s/\s+//;
+ my $sf = $2;
+ $sf =~ s/\s+//;
+ my $match = $3;
+ $match =~ s/^\s*//; $match =~ s/\s*$//;
+ $fields{$field} = { sf => [ split('', $sf) ] };
+ if ($match) {
+ my ($msf,$mre) = split('~', $match);
+ if (length($msf) > 0 and length($mre) > 0) {
+ $msf =~ s/^\s*//; $msf =~ s/\s*$//;
+ $mre =~ s/^\s*//; $mre =~ s/\s*$//;
+ $fields{$field}{match} = { sf => $msf, re => qr/$mre/ };
+ }
+ }
+ }
+ }
+
+ # Returns a flat list of subfield (code, value, code, value, ...)
+ # suitable for adding to a MARC::Field.
+ sub generate_replacement_subfields {
+ my ($source_field, $target_field, @controlled_subfields) = @_;
+
+ # Performing a wholesale field replacment.
+ # Use the entire source field as-is.
+ return map {$_->[0], $_->[1]} $source_field->subfields
+ unless @controlled_subfields;
+
+ my @new_subfields;
+
+ # Iterate over all target field subfields:
+ # 1. Keep uncontrolled subfields as is.
+ # 2. Replace values for controlled subfields when a
+ # replacement value exists on the source record.
+ # 3. Delete values for controlled subfields when no
+ # replacement value exists on the source record.
+
+ for my $target_sf ($target_field->subfields) {
+ my $subfield = $target_sf->[0];
+ my $target_val = $target_sf->[1];
+
+ if (grep {$_ eq $subfield} @controlled_subfields) {
+ if (my $source_val = $source_field->subfield($subfield)) {
+ # We have a replacement value
+ push(@new_subfields, $subfield, $source_val);
+ } else {
+ # no replacement value for controlled subfield, drop it.
+ }
+ } else {
+ # Field is not controlled. Copy it over as-is.
+ push(@new_subfields, $subfield, $target_val);
+ }
+ }
+
+ # Iterate over all subfields in the source field and back-fill
+ # any values that exist only in the source field. Insert these
+ # subfields in the same relative position they exist in the
+ # source field.
+
+ my @seen_subfields;
+ for my $source_sf ($source_field->subfields) {
+ my $subfield = $source_sf->[0];
+ my $source_val = $source_sf->[1];
+ push(@seen_subfields, $subfield);
+
+ # target field already contains this subfield,
+ # so it would have been addressed above.
+ next if $target_field->subfield($subfield);
+
+ # Ignore uncontrolled subfields.
+ next unless grep {$_ eq $subfield} @controlled_subfields;
+
+ # Adding a new subfield. Find its relative position and add
+ # it to the list under construction. Work backwards from
+ # the list of already seen subfields to find the best slot.
+
+ my $done = 0;
+ for my $seen_sf (reverse(@seen_subfields)) {
+ my $idx = @new_subfields;
+ for my $new_sf (reverse(@new_subfields)) {
+ $idx--;
+ next if $idx % 2 == 1; # sf codes are in the even slots
+
+ if ($new_subfields[$idx] eq $seen_sf) {
+ splice(@new_subfields, $idx + 2, 0, $subfield, $source_val);
+ $done = 1;
+ last;
+ }
+ }
+ last if $done;
+ }
+
+ # if no slot was found, add to the end of the list.
+ push(@new_subfields, $subfield, $source_val) unless $done;
+ }
+
+ return @new_subfields;
+ }
+
+ # MARC tag loop
+ for my $f (keys %fields) {
+ my $tag_idx = -1;
+ for my $target_field ($target_r->field($f)) {
+
+ # field spec contains a regex for this field. Confirm field on
+ # target record matches the specified regex before replacing.
+ if (exists($fields{$f}{match})) {
+ next unless (grep { $_ =~ $fields{$f}{match}{re} }
+ $target_field->subfield($fields{$f}{match}{sf}));
+ }
+
+ my @new_subfields;
+ my @controlled_subfields = @{$fields{$f}{sf}};
+
+ # If the target record has multiple matching bib fields,
+ # replace them from matching fields on the source record
+ # in a predictable order to avoid replacing with them with
+ # same source field repeatedly.
+ my @source_fields = $source_r->field($f);
+ my $source_field = $source_fields[++$tag_idx];
+
+ if (!$source_field && @controlled_subfields) {
+ # When there are more target fields than source fields
+ # and we are replacing values for subfields and not
+ # performing wholesale field replacment, use the last
+ # available source field as the input for all remaining
+ # target fields.
+ $source_field = $source_fields[$#source_fields];
+ }
+
+ if (!$source_field) {
+ # No source field exists. Delete all affected target
+ # data. This is a little bit counterintuitive, but is
+ # backwards compatible with the previous version of this
+ # function which first deleted all affected data, then
+ # replaced values where possible.
+ if (@controlled_subfields) {
+ $target_field->delete_subfield($_) for @controlled_subfields;
+ } else {
+ $target_r->delete_field($target_field);
+ }
+ next;
+ }
+
+ my @new_subfields = generate_replacement_subfields(
+ $source_field, $target_field, @controlled_subfields);
+
+ # Build the replacement field from scratch.
+ my $replacement_field = MARC::Field->new(
+ $target_field->tag,
+ $target_field->indicator(1),
+ $target_field->indicator(2),
+ @new_subfields
+ );
+
+ $target_field->replace_with($replacement_field);
+ }
+ }
+
+ $target_xml = $target_r->as_xml_record;
+ $target_xml =~ s/^<\?.+?\?>$//mo;
+ $target_xml =~ s/\n//sgo;
+ $target_xml =~ s/>\s+</></sgo;
+
+ return $target_xml;
+
+$_$ LANGUAGE PLPERLU;
+
+COMMIT;
+
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/012.schema.vandelay.sql | 210 ++++++++++++++++++---
.../sql/Pg/t/lp712490-vand-replace-field-merge.pg | 135 +++++++++++++
.../1157.schema.vandelay-replace-field-order.sql | 202 ++++++++++++++++++++
4 files changed, 526 insertions(+), 23 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/t/lp712490-vand-replace-field-merge.pg
create mode 100644 Open-ILS/src/sql/Pg/upgrade/1157.schema.vandelay-replace-field-order.sql
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list