[open-ils-commits] r11401 - in trunk/Open-ILS/src/perlmods/OpenILS/Application: . Cat

[svn at svn.open-ils.org]

Author: miker
Date: 2008-12-04 00:30:32 -0500 (Thu, 04 Dec 2008)
New Revision: 11401

Modified:
   trunk/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm
   trunk/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
Log:
queue up ingest requests in a secondary stream to avoid piling them up and exhausting cstore sessions

Modified: trunk/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm	2008-12-03 23:03:57 UTC (rev 11400)
+++ trunk/Open-ILS/src/perlmods/OpenILS/Application/Cat/BibCommon.pm	2008-12-04 05:30:32 UTC (rev 11401)
@@ -36,7 +36,7 @@
 
 
 sub biblio_record_replace_marc  {
-	my($class, $e, $recid, $newxml, $source, $fixtcn, $override) = @_;
+	my($class, $e, $recid, $newxml, $source, $fixtcn, $override, $noingest) = @_;
 
 	my $rec = $e->retrieve_biblio_record_entry($recid)
 		or return $e->die_event;
@@ -70,15 +70,17 @@
 	$rec->marc( $U->entityize( $marcdoc->documentElement->toString ) );
 	$e->update_biblio_record_entry($rec) or return $e->die_event;
 
-    # we don't care about the result, just fire off the request
-    my $ses = OpenSRF::AppSession->create('open-ils.ingest');
-    $ses->request('open-ils.ingest.full.biblio.record', $recid);
+    unless ($noingest) {
+        # we don't care about the result, just fire off the request
+        my $ses = OpenSRF::AppSession->create('open-ils.ingest');
+        $ses->request('open-ils.ingest.full.biblio.record', $recid);
+    }
 
 	return $rec;
 }
 
 sub biblio_record_xml_import {
-	my($class, $e, $xml, $source, $auto_tcn, $override) = @_;
+	my($class, $e, $xml, $source, $auto_tcn, $override, $noingest) = @_;
 
 	my( $evt, $tcn, $tcn_source, $marcdoc );
 
@@ -107,9 +109,11 @@
     $record = $e->create_biblio_record_entry($record) or return $e->die_event;
 	$logger->info("marc create/import created new record ".$record->id);
 
-    # we don't care about the result, just fire off the request
-    my $ses = OpenSRF::AppSession->create('open-ils.ingest');
-    $ses->request('open-ils.ingest.full.biblio.record', $record->id);
+    unless ($noingest) {
+        # we don't care about the result, just fire off the request
+        my $ses = OpenSRF::AppSession->create('open-ils.ingest');
+        $ses->request('open-ils.ingest.full.biblio.record', $record->id);
+    }
 
 	return $record;
 }

Modified: trunk/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm	2008-12-03 23:03:57 UTC (rev 11400)
+++ trunk/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm	2008-12-04 05:30:32 UTC (rev 11401)
@@ -534,7 +534,10 @@
     my $total = @$rec_ids;
     my $count = 0;
     my %queues;
+    my @ingest_queue;
 
+    my $ingest_ses = OpenSRF::AppSession->connect('open-ils.ingest');
+
     for my $rec_id (@$rec_ids) {
 
         my $overlay_target = $overlay_map->{$rec_id};
@@ -562,11 +565,11 @@
             if(defined $overlay_target) {
                 $logger->info("vl: overlaying record $overlay_target");
                 $record = OpenILS::Application::Cat::BibCommon->biblio_record_replace_marc(
-                    $e, $overlay_target, $rec->marc); #$rec->bib_source
+                    $e, $overlay_target, $rec->marc, undef, undef, undef, 1); #$rec->bib_source
             } else {
                 $logger->info("vl: importing new record");
                 $record = OpenILS::Application::Cat::BibCommon->biblio_record_xml_import(
-                    $e, $rec->marc); #$rec->bib_source
+                    $e, $rec->marc, undef, undef, undef, 1); #$rec->bib_source
             }
 
             if($U->event_code($record)) {
@@ -583,6 +586,10 @@
                 next;
             }
 
+            $e->commit;
+            push @ingest_queue, $ingest_ses->request('open-ils.ingest.full.biblio.record', $rec_id);
+            push @ingest_queue, { req => $ingest_ses->request('open-ils.ingest.full.biblio.record', $rec_id), rec_id => $rec_id };
+
         } else { # authority
 
             my $rec = $e->retrieve_vandelay_queued_authority_record($rec_id);
@@ -623,9 +630,11 @@
                 $e->rollback;
                 next;
             }
+
+            $e->commit;
+            push @ingest_queue, { req => $ingest_ses->request('open-ils.ingest.full.authority.record', $rec_id), rec_id => $rec_id };
         }
 
-        $e->commit;
         $conn->respond({total => $total, progress => $count, imported => $rec_id}) if (++$count % 10) == 0;
     }
 
@@ -641,7 +650,7 @@
                     $queue->complete('t');
                     $e->update_vandelay_bib_queue($queue) or return $e->die_event;
                     $e->commit;
-                    return;
+                    last
                 }
             }
         } else {
@@ -653,12 +662,20 @@
                     $queue->complete('t');
                     $e->update_vandelay_authority_queue($queue) or return $e->die_event;
                     $e->commit;
-                    return;
+                    last
                 }
             }
         }
     }
 
+    $count = 0;
+    for my $ingest (@ingest_queue) {
+        try { $ingest->{req}->gather(1); } otherwise {};
+        $conn->respond({total => $total, progress => $count, imported => $ingest->{rec_id}}) if (++$count % 10) == 0;
+    } 
+
+    $ingest_ses->disconnect();
+
     $e->rollback;
     return undef;
 }