[open-ils-commits] r15294 - in branches/rel_1_6_0/Open-ILS: src/extras/import src/perlmods/OpenILS/Application tests/datasets (dbs)
svn at svn.open-ils.org
svn at svn.open-ils.org
Sun Jan 10 19:49:51 EST 2010
Author: dbs
Date: 2010-01-10 19:49:50 -0500 (Sun, 10 Jan 2010)
New Revision: 15294
Modified:
branches/rel_1_6_0/Open-ILS/src/extras/import/direct_ingest.pl
branches/rel_1_6_0/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
branches/rel_1_6_0/Open-ILS/tests/datasets/README
Log:
Remove serial ingest references and services
The references and services for serial (MFHD) support are not
necessary in 1.6, as we are working directly with the MFHD as
maintained in serial.record_entry.marc; the services and
references being removed never could have been used in the
1.6.0.0 release and should have been removed prior to the 1.6.0.0
release when I removed the corresponding database structures.
Also updating the README to reflect the proper loading method for
serial records, dropping direct_ingest.pl invocation and noting the
limitation on the org_unit identification.
Modified: branches/rel_1_6_0/Open-ILS/src/extras/import/direct_ingest.pl
===================================================================
--- branches/rel_1_6_0/Open-ILS/src/extras/import/direct_ingest.pl 2010-01-11 00:49:28 UTC (rev 15293)
+++ branches/rel_1_6_0/Open-ILS/src/extras/import/direct_ingest.pl 2010-01-11 00:49:50 UTC (rev 15294)
@@ -26,13 +26,12 @@
MARC::Charset->ignore_errors(1);
-my ($max_uri, $max_cn, $auth, $mfhd, $config, $quiet) =
- (0, 0, 0, 0, '/openils/conf/opensrf_core.xml');
+my ($max_uri, $max_cn, $auth, $config, $quiet) =
+ (0, 0, 0, '/openils/conf/opensrf_core.xml');
GetOptions(
'config=s' => \$config,
'authority' => \$auth,
- 'serial' => \$mfhd,
'quiet' => \$quiet,
'max_uri=i' => \$max_uri,
'max_cn=i' => \$max_cn,
@@ -53,7 +52,6 @@
my $meth = 'open-ils.ingest.full.biblio.object.readonly';
$meth = 'open-ils.ingest.full.authority.object.readonly' if ($auth);
-$meth = 'open-ils.ingest.full.serial.object.readonly' if ($mfhd);
$meth = OpenILS::Application::Ingest->method_lookup( $meth );
@@ -91,7 +89,7 @@
my $bib = $data->{bib};
my $full_rec = $data->{ingest_data}->{full_rec};
- if (!$auth && !$mfhd) {
+ if (!$auth) {
$field_entries = $data->{ingest_data}->{field_entries};
$fp = $data->{ingest_data}->{fingerprint};
$rd = $data->{ingest_data}->{descriptor};
@@ -102,7 +100,7 @@
}
print( OpenSRF::Utils::JSON->perl2JSON($bib)."\n" );
- if (!$auth && !$mfhd) {
+ if (!$auth) {
print( OpenSRF::Utils::JSON->perl2JSON($rd)."\n" );
print( OpenSRF::Utils::JSON->perl2JSON($_)."\n" ) for (@$field_entries);
for my $u (@$uri) {
Modified: branches/rel_1_6_0/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
===================================================================
--- branches/rel_1_6_0/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm 2010-01-11 00:49:28 UTC (rev 15293)
+++ branches/rel_1_6_0/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm 2010-01-11 00:49:50 UTC (rev 15294)
@@ -819,168 +819,6 @@
);
# --------------------------------------------------------------------------------
-# Serial ingest
-
-package OpenILS::Application::Ingest::Serial;
-use base qw/OpenILS::Application::Ingest/;
-use Unicode::Normalize;
-
-sub ro_serial_ingest_single_object {
- my $self = shift;
- my $client = shift;
- my $bib = shift;
- my $xml = OpenILS::Application::AppUtils->entityize($bib->marc);
-
- my $document = $parser->parse_string($xml);
-
- my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document);
-
- $_->record($bib->id) for (@mfr);
-
- return { full_rec => \@mfr };
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.object.readonly",
- method => "ro_serial_ingest_single_object",
- api_level => 1,
- argc => 1,
-);
-
-sub ro_serial_ingest_single_xml {
- my $self = shift;
- my $client = shift;
- my $xml = OpenILS::Application::AppUtils->entityize(shift);
-
- my $document = $parser->parse_string($xml);
-
- my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.serial.xml")->run($document);
-
- return { full_rec => \@mfr };
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.xml.readonly",
- method => "ro_serial_ingest_single_xml",
- api_level => 1,
- argc => 1,
-);
-
-sub ro_serial_ingest_single_record {
- my $self = shift;
- my $client = shift;
- my $rec = shift;
-
- OpenILS::Application::Ingest->post_init();
- my $r = OpenSRF::AppSession
- ->create('open-ils.cstore')
- ->request( 'open-ils.cstore.direct.serial.record_entry.retrieve' => $rec )
- ->gather(1);
-
- return undef unless ($r and @$r);
-
- my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($r->marc);
-
- $_->record($rec) for (@{$res->{full_rec}});
- $res->{descriptor}->record($rec);
-
- return $res;
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.record.readonly",
- method => "ro_serial_ingest_single_record",
- api_level => 1,
- argc => 1,
-);
-
-sub ro_serial_ingest_stream_record {
- my $self = shift;
- my $client = shift;
-
- OpenILS::Application::Ingest->post_init();
-
- my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
- while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-
- my $rec = $resp->content;
- last unless (defined $rec);
-
- $log->debug("Running open-ils.ingest.full.serial.record.readonly ...");
- my ($res) = $self->method_lookup("open-ils.ingest.full.serial.record.readonly")->run($rec);
-
- $_->record($rec) for (@{$res->{full_rec}});
-
- $client->respond( $res );
- }
-
- return undef;
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.record_stream.readonly",
- method => "ro_serial_ingest_stream_record",
- api_level => 1,
- stream => 1,
-);
-
-sub ro_serial_ingest_stream_xml {
- my $self = shift;
- my $client = shift;
-
- OpenILS::Application::Ingest->post_init();
-
- my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
- while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-
- my $xml = $resp->content;
- last unless (defined $xml);
-
- $log->debug("Running open-ils.ingest.full.serial.xml.readonly ...");
- my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($xml);
-
- $client->respond( $res );
- }
-
- return undef;
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.xml_stream.readonly",
- method => "ro_serial_ingest_stream_xml",
- api_level => 1,
- stream => 1,
-);
-
-sub rw_serial_ingest_stream_import {
- my $self = shift;
- my $client = shift;
-
- OpenILS::Application::Ingest->post_init();
-
- my $ses = OpenSRF::AppSession->create('open-ils.cstore');
-
- while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
-
- my $bib = $resp->content;
- last unless (defined $bib);
-
- $log->debug("Running open-ils.ingest.full.serial.xml.readonly ...");
- my ($res) = $self->method_lookup("open-ils.ingest.full.serial.xml.readonly")->run($bib->marc);
-
- $_->record($bib->id) for (@{$res->{full_rec}});
-
- $client->respond( $res );
- }
-
- return undef;
-}
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.full.serial.bib_stream.import",
- method => "rw_serial_ingest_stream_import",
- api_level => 1,
- stream => 1,
-);
-
-
-# --------------------------------------------------------------------------------
# MARC index extraction
package OpenILS::Application::Ingest::XPATH;
@@ -1270,7 +1108,6 @@
my $type = 'metabib';
$type = 'authority' if ($self->api_name =~ /authority/o);
- $type = 'serial' if ($self->api_name =~ /serial/o);
OpenILS::Application::Ingest->post_init();
@@ -1291,13 +1128,6 @@
argc => 1,
stream => 1,
);
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.flat_marc.serial.xml",
- method => "flat_marc_xml",
- api_level => 1,
- argc => 1,
- stream => 1,
-);
sub flat_marc_record {
my $self = shift;
@@ -1306,7 +1136,6 @@
my $type = 'biblio';
$type = 'authority' if ($self->api_name =~ /authority/o);
- $type = 'serial' if ($self->api_name =~ /serial/o);
OpenILS::Application::Ingest->post_init();
my $r = OpenSRF::AppSession
@@ -1338,13 +1167,6 @@
argc => 1,
stream => 1,
);
-__PACKAGE__->register_method(
- api_name => "open-ils.ingest.flat_marc.serial.record_entry",
- method => "flat_marc_record",
- api_level => 1,
- argc => 1,
- stream => 1,
-);
# --------------------------------------------------------------------------------
# URI extraction
Modified: branches/rel_1_6_0/Open-ILS/tests/datasets/README
===================================================================
--- branches/rel_1_6_0/Open-ILS/tests/datasets/README 2010-01-11 00:49:28 UTC (rev 15293)
+++ branches/rel_1_6_0/Open-ILS/tests/datasets/README 2010-01-11 00:49:50 UTC (rev 15294)
@@ -19,7 +19,14 @@
- First load the MARC21 records:
- PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2bre.pl --marctype XML --start 1 --idfield 901 --idsubfield a serials_marc21.xml | perl ../../src/extras/import/direct_ingest.pl | perl ../../src/extras/import/pg_loader.pl -or bre -or mrd -or mfr -or mtfe -or mafe -or msfe -or mkfe -or msefe -a mrd -a mfr -a mtfe -a mafe -a msfe -a mkfe -a msefe | psql -U evergreen -h localhost
- - Then load the MFHD records
- - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password demo123 serials_mfhd.xml | perl ../../src/extras/import/direct_ingest.pl --serial | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql
+ - Then generate the metarecord map:
+ - pgsql -f ../../src/extras/import/quick_metarecord_map.sql
+
+ - Then process and load the MFHD records - ingest is not used:
+ - PERL5LIB=/openils/lib/perl5/ perl ../../src/extras/import/marc2sre.pl --marctype XML --password open-ils serials_mfhd.xml | perl ../../src/extras/import/pg_loader.pl -or sre > mfhd21.sql
+ - Open the file in a text editor and change the final "\N" on each line
+ to the numeric ID of the org_unit that owns the MFHD record. Yes, this
+ is a gross hack that effectively limits you to loading MFHD records in
+ per-org_unit batches; enhancements to marc2sre.pl are welcomed.
- psql -f mfhd21.sql
More information about the open-ils-commits
mailing list