[open-ils-commits] r18820 - in branches/rel_2_0/Open-ILS: examples src/perlmods/OpenILS/Application/Storage/Publisher src/support-scripts (gmc)

[svn at svn.open-ils.org]

Author: gmc
Date: 2010-11-22 08:38:53 -0500 (Mon, 22 Nov 2010)
New Revision: 18820

Modified:
   branches/rel_2_0/Open-ILS/examples/opensrf.xml.example
   branches/rel_2_0/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
   branches/rel_2_0/Open-ILS/src/support-scripts/fine_generator.pl
Log:
parallel fine generator

The fine generator cronjob can now use multiple
parallel processes by setting fine_generator/parallel
in opensrf.xml to a value greater than 1.  This
can speed up periodic fine generation in a database
containing a large number of overdue loans.

Also added a service to return just the list of
IDs of overdue loans and reservations - fleshing
the entire set of overdue loans when generating fines
has been observed to cause significant swap-thrashing in
at least one large database.

Signed-off-by: Galen Charlton <gmc at esilibrary.com>

Modified: branches/rel_2_0/Open-ILS/examples/opensrf.xml.example
===================================================================
--- branches/rel_2_0/Open-ILS/examples/opensrf.xml.example	2010-11-22 13:38:51 UTC (rev 18819)
+++ branches/rel_2_0/Open-ILS/examples/opensrf.xml.example	2010-11-22 13:38:53 UTC (rev 18820)
@@ -139,6 +139,12 @@
             -->
             <parallel>1</parallel>
         </hold_targeter>
+
+        <!-- Settings for the fine generator cron job -->
+        <fine_generator>
+            <!-- number of parallel processes to use during fine generation -->
+            <parallel>1</parallel>
+        </fine_generator>
         
         <reporter>
             <!--

Modified: branches/rel_2_0/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm
===================================================================
--- branches/rel_2_0/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm	2010-11-22 13:38:51 UTC (rev 18819)
+++ branches/rel_2_0/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/action.pm	2010-11-22 13:38:53 UTC (rev 18820)
@@ -94,6 +94,7 @@
 sub overdue_circs {
 	my $grace = shift;
     my $upper_interval = shift || '1 millennium';
+	my $idlist = shift;
 
 	my $c_t = action::circulation->table;
 
@@ -114,7 +115,7 @@
 	my $sth = action::circulation->db_Main->prepare_cached($sql);
 	$sth->execute($upper_interval);
 
-	my @circs = map { action::circulation->construct($_) } $sth->fetchall_hash;
+	my @circs = map { $idlist ? $_->{id} : action::circulation->construct($_) } $sth->fetchall_hash;
 
 	$c_t = booking::reservation->table;
 	$sql = <<"	SQL";
@@ -129,7 +130,7 @@
 	$sth = action::circulation->db_Main->prepare_cached($sql);
 	$sth->execute();
 
-    push @circs, map { booking::reservation->construct($_) } $sth->fetchall_hash;
+    push @circs, map { $idlist ? $_->{id} : booking::reservation->construct($_) } $sth->fetchall_hash;
 
     return @circs;
 }
@@ -264,7 +265,9 @@
 	my $client = shift;
 	my $grace = shift || '';
 
-	$client->respond( $_->to_fieldmapper ) for ( overdue_circs($grace) );
+	my $idlist = $self->api_name =~/id_list/o ? 1 : 0;
+    
+	$client->respond( $idlist ? $_ : $_->to_fieldmapper ) for ( overdue_circs($grace, '', $idlist) );
 
 	return undef;
 
@@ -275,6 +278,12 @@
 	stream		=> 1,
 	method          => 'grab_overdue',
 );
+__PACKAGE__->register_method(
+	api_name        => 'open-ils.storage.action.circulation.overdue.id_list',
+	api_level       => 1,
+	stream		=> 1,
+	method          => 'grab_overdue',
+);
 
 sub nearest_hold {
 	my $self = shift;

Modified: branches/rel_2_0/Open-ILS/src/support-scripts/fine_generator.pl
===================================================================
--- branches/rel_2_0/Open-ILS/src/support-scripts/fine_generator.pl	2010-11-22 13:38:51 UTC (rev 18819)
+++ branches/rel_2_0/Open-ILS/src/support-scripts/fine_generator.pl	2010-11-22 13:38:53 UTC (rev 18820)
@@ -8,6 +8,8 @@
 use warnings;
 use OpenSRF::Utils::JSON;
 use OpenSRF::System;
+use OpenSRF::Utils::SettingsClient;
+use OpenSRF::MultiSession;
 
 my $config = shift || die "bootstrap config required\n";
 my $lockfile = shift || "/tmp/generate_fines-LOCK";
@@ -35,11 +37,35 @@
 close F;
 
 OpenSRF::System->bootstrap_client( config_file => $config );
+my $settings = OpenSRF::Utils::SettingsClient->new;
+my $parallel = $settings->config_value( fine_generator => 'parallel' ) || 1; 
 
-my $r = OpenSRF::AppSession
-		->create( 'open-ils.storage' )
-		->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
+if ($parallel == 1) {
 
-while (!$r->complete) { $r->recv };
+    my $r = OpenSRF::AppSession
+            ->create( 'open-ils.storage' )
+            ->request( 'open-ils.storage.action.circulation.overdue.generate_fines' => $grace );
 
+    while (!$r->complete) { $r->recv };
+
+} else {
+
+    my $multi_generator = OpenSRF::MultiSession->new(
+        app => 'open-ils.storage', 
+        cap => $parallel, 
+        api_level => 1,
+    );
+
+    my $storage = OpenSRF::AppSession->create("open-ils.storage");
+    my $r = $storage->request('open-ils.storage.action.circulation.overdue.id_list', $grace);
+    while (my $resp = $r->recv) {
+        my $circ_id = $resp->content;
+        $multi_generator->request( 'open-ils.storage.action.circulation.overdue.generate_fines', $grace, $circ_id );
+    }
+    $storage->disconnect();
+    $multi_generator->session_wait(1);
+    $multi_generator->disconnect;
+
+}
+
 unlink $lockfile;