[open-ils-commits] r7740 - in
trunk/Open-ILS/src/perlmods/OpenILS/WWW: . AddedContent
svn at svn.open-ils.org
svn at svn.open-ils.org
Wed Aug 29 16:21:59 EDT 2007
Author: erickson
Date: 2007-08-29 16:16:04 -0400 (Wed, 29 Aug 2007)
New Revision: 7740
Modified:
trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm
trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm
Log:
moved image fetching into the syndetics added content plugin. added a caching layer to all added content
Modified: trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm 2007-08-29 16:11:11 UTC (rev 7739)
+++ trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm 2007-08-29 20:16:04 UTC (rev 7740)
@@ -1,232 +1,262 @@
package OpenILS::WWW::AddedContent::Syndetic;
use strict; use warnings;
-use LWP::UserAgent;
use OpenSRF::Utils::Logger qw/$logger/;
use OpenSRF::Utils::SettingsParser;
use OpenSRF::Utils::JSON;
use OpenSRF::EX qw/:try/;
use OpenILS::WWW::AddedContent;
+use XML::LibXML;
+use MIME::Base64;
+my $AC = 'OpenILS::WWW::AddedContent';
sub new {
- my( $class, $args ) = @_;
- $class = ref $class || $class;
- return bless($args, $class);
+ my( $class, $args ) = @_;
+ $class = ref $class || $class;
+ return bless($args, $class);
}
sub base_url {
- my $self = shift;
- return $self->{base_url};
+ my $self = shift;
+ return $self->{base_url};
}
sub userid {
- my $self = shift;
- return $self->{userid};
+ my $self = shift;
+ return $self->{userid};
}
# --------------------------------------------------------------------------
+sub jacket_small {
+ my( $self, $key ) = @_;
+ return $self->send_img(
+ $self->fetch_response('sc.gif', $key, 1));
+}
+sub jacket_medium {
+ my( $self, $key ) = @_;
+ return $self->send_img(
+ $self->fetch_response('mc.gif', $key, 1));
+
+}
+sub jacket_large {
+ my( $self, $key ) = @_;
+ return $self->send_img(
+ $self->fetch_response('lc.gif', $key, 1));
+}
+
+# --------------------------------------------------------------------------
+
sub toc_html {
- my( $self, $key ) = @_;
- return $self->send_html(
- $self->fetch_content('toc.html', $key));
+ my( $self, $key ) = @_;
+ return $self->send_html(
+ $self->fetch_content('toc.html', $key));
}
sub toc_xml {
- my( $self, $key ) = @_;
- return $self->send_xml(
- $self->fetch_content('toc.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_xml(
+ $self->fetch_content('toc.xml', $key));
}
sub toc_json {
- my( $self, $key ) = @_;
- return $self->send_json(
- $self->fetch_content('toc.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_json(
+ $self->fetch_content('toc.xml', $key));
}
# --------------------------------------------------------------------------
sub anotes_html {
- my( $self, $key ) = @_;
- return $self->send_html(
- $self->fetch_content('anotes.html', $key));
+ my( $self, $key ) = @_;
+ return $self->send_html(
+ $self->fetch_content('anotes.html', $key));
}
sub anotes_xml {
- my( $self, $key ) = @_;
- return $self->send_xml(
- $self->fetch_content('anotes.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_xml(
+ $self->fetch_content('anotes.xml', $key));
}
sub anotes_json {
- my( $self, $key ) = @_;
- return $self->send_json(
- $self->fetch_content('anotes.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_json(
+ $self->fetch_content('anotes.xml', $key));
}
# --------------------------------------------------------------------------
sub excerpt_html {
- my( $self, $key ) = @_;
- return $self->send_html(
- $self->fetch_content('dbchapter.html', $key));
+ my( $self, $key ) = @_;
+ return $self->send_html(
+ $self->fetch_content('dbchapter.html', $key));
}
sub excerpt_xml {
- my( $self, $key ) = @_;
- return $self->send_xml(
- $self->fetch_content('dbchapter.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_xml(
+ $self->fetch_content('dbchapter.xml', $key));
}
sub excerpt_json {
- my( $self, $key ) = @_;
- return $self->send_json(
- $self->fetch_content('dbchapter.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_json(
+ $self->fetch_content('dbchapter.xml', $key));
}
# --------------------------------------------------------------------------
sub reviews_html {
- my( $self, $key ) = @_;
+ my( $self, $key ) = @_;
- my %reviews;
+ my %reviews;
- $reviews{ljreview} = $self->fetch_content('ljreview.html', $key);
- $reviews{pwreview} = $self->fetch_content('pwreview.html', $key);
- $reviews{slreview} = $self->fetch_content('slreview.html', $key);
- $reviews{chreview} = $self->fetch_content('chreview.html', $key);
- $reviews{blreview} = $self->fetch_content('blreview.html', $key);
- $reviews{hbreview} = $self->fetch_content('hbreview.html', $key);
- $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key);
+ $reviews{ljreview} = $self->fetch_content('ljreview.html', $key);
+ $reviews{pwreview} = $self->fetch_content('pwreview.html', $key);
+ $reviews{slreview} = $self->fetch_content('slreview.html', $key);
+ $reviews{chreview} = $self->fetch_content('chreview.html', $key);
+ $reviews{blreview} = $self->fetch_content('blreview.html', $key);
+ $reviews{hbreview} = $self->fetch_content('hbreview.html', $key);
+ $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key);
- for(keys %reviews) {
- if( ! $self->data_exists($reviews{$_}) ) {
- delete $reviews{$_};
- next;
- }
- $reviews{$_} =~ s/<!.*?>//og; # Strip any doctype declarations
- }
+ for(keys %reviews) {
+ if( ! $self->data_exists($reviews{$_}) ) {
+ delete $reviews{$_};
+ next;
+ }
+ $reviews{$_} =~ s/<!.*?>//og; # Strip any doctype declarations
+ }
- return 0 if scalar(keys %reviews) == 0;
-
- #my $html = "<div>";
- my $html;
- $html .= $reviews{$_} for keys %reviews;
- #$html .= "</div>";
+ return 0 if scalar(keys %reviews) == 0;
+
+ #my $html = "<div>";
+ my $html;
+ $html .= $reviews{$_} for keys %reviews;
+ #$html .= "</div>";
- return $self->send_html($html);
+ return $self->send_html($html);
}
# we have to aggregate the reviews
sub reviews_xml {
- my( $self, $key ) = @_;
- my %reviews;
+ my( $self, $key ) = @_;
+ my %reviews;
- $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key);
- $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key);
- $reviews{slreview} = $self->fetch_content('slreview.xml', $key);
- $reviews{chreview} = $self->fetch_content('chreview.xml', $key);
- $reviews{blreview} = $self->fetch_content('blreview.xml', $key);
- $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key);
- $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key);
+ $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key);
+ $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key);
+ $reviews{slreview} = $self->fetch_content('slreview.xml', $key);
+ $reviews{chreview} = $self->fetch_content('chreview.xml', $key);
+ $reviews{blreview} = $self->fetch_content('blreview.xml', $key);
+ $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key);
+ $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key);
- for(keys %reviews) {
- if( ! $self->data_exists($reviews{$_}) ) {
- delete $reviews{$_};
- next;
- }
- # Strip the xml and doctype declarations
- $reviews{$_} =~ s/<\?xml.*?>//og;
- $reviews{$_} =~ s/<!.*?>//og;
- }
+ for(keys %reviews) {
+ if( ! $self->data_exists($reviews{$_}) ) {
+ delete $reviews{$_};
+ next;
+ }
+ # Strip the xml and doctype declarations
+ $reviews{$_} =~ s/<\?xml.*?>//og;
+ $reviews{$_} =~ s/<!.*?>//og;
+ }
- return 0 if scalar(keys %reviews) == 0;
-
- my $xml = "<reviews>";
- $xml .= $reviews{$_} for keys %reviews;
- $xml .= "</reviews>";
+ return 0 if scalar(keys %reviews) == 0;
+
+ my $xml = "<reviews>";
+ $xml .= $reviews{$_} for keys %reviews;
+ $xml .= "</reviews>";
- return $self->send_xml($xml);
+ return $self->send_xml($xml);
}
sub reviews_json {
- my( $self, $key ) = @_;
- return $self->send_json(
- $self->fetch_content('dbchapter.xml', $key));
+ my( $self, $key ) = @_;
+ return $self->send_json(
+ $self->fetch_content('dbchapter.xml', $key));
}
# --------------------------------------------------------------------------
sub data_exists {
- my( $self, $data ) = @_;
- return 0 if $data =~ m/<title>error<\/title>/iog;
- return 1;
+ my( $self, $data ) = @_;
+ return 0 if $data =~ m/<title>error<\/title>/iog;
+ return 1;
}
sub send_json {
- my( $self, $xml ) = @_;
- return 0 unless $self->data_exists($xml);
- my $doc;
+ my( $self, $xml ) = @_;
+ return 0 unless $self->data_exists($xml);
+ my $doc;
- try {
- $doc = XML::LibXML->new->parse_string($xml);
- } catch Error with {
- my $err = shift;
- $logger->error("added content XML parser error: $err\n\n$xml");
- $doc = undef;
- };
+ try {
+ $doc = XML::LibXML->new->parse_string($xml);
+ } catch Error with {
+ my $err = shift;
+ $logger->error("added content XML parser error: $err\n\n$xml");
+ $doc = undef;
+ };
- return 0 unless $doc;
- my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement);
- my $json = OpenSRF::Utils::JSON->perl2JSON($perl);
- print "Content-type: text/plain\n\n";
- print $json;
- return 1;
+ return 0 unless $doc;
+ my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement);
+ my $json = OpenSRF::Utils::JSON->perl2JSON($perl);
+ return { content_type => 'text/plain', content => $json };
}
sub send_xml {
- my( $self, $xml ) = @_;
- return 0 unless $self->data_exists($xml);
- print "Content-Type: application/xml\n\n";
- print $xml;
- return 1;
+ my( $self, $xml ) = @_;
+ return 0 unless $self->data_exists($xml);
+ return { content_type => 'application/xml', content => $xml };
}
sub send_html {
- my( $self, $content ) = @_;
- return 0 unless $self->data_exists($content);
+ my( $self, $content ) = @_;
+ return 0 unless $self->data_exists($content);
- # Hide anything that might contain a link since it will be broken
- my $HTML = <<" HTML";
- <div>
- <style type='text/css'>
- div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden }
- </style>
- <div class='ac'>
- $content
- </div>
- </div>
- HTML
+ # Hide anything that might contain a link since it will be broken
+ my $HTML = <<" HTML";
+ <div>
+ <style type='text/css'>
+ div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden }
+ </style>
+ <div class='ac'>
+ $content
+ </div>
+ </div>
+ HTML
- print "Content-type: text/html\n\n";
- print $HTML;
+ return { content_type => 'text/html', content => $HTML };
+}
- return 1;
+sub send_img {
+ my($self, $response) = @_;
+ return {
+ content_type => $response->header('Content-type'),
+ content => $response->content,
+ binary => 1
+ };
}
+# returns the raw content returned from the URL fetch
sub fetch_content {
- my( $self, $page, $key ) = @_;
- my $uname = $self->userid;
- my $url = $self->base_url . "?isbn=$key/$page&client=$uname&type=rw12";
- return OpenILS::WWW::AddedContent->get_url($url);
+ my( $self, $page, $key ) = @_;
+ return $self->fetch_response($page, $key)->content;
}
+# returns the HTTP response object from the URL fetch
+sub fetch_response {
+ my( $self, $page, $key, $notype ) = @_;
+ my $uname = $self->userid;
+ my $url = $self->base_url . "?isbn=$key/$page&client=$uname" . (($notype) ? '' : "&type=rw12");
+ return $AC->get_url($url);
+}
+
1;
Modified: trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm 2007-08-29 16:11:11 UTC (rev 7739)
+++ trunk/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm 2007-08-29 20:16:04 UTC (rev 7740)
@@ -21,10 +21,11 @@
use LWP::UserAgent;
use MIME::Base64;
+my $AC = __PACKAGE__;
+
# set the bootstrap config when this module is loaded
my $bs_config;
-my $handler;
sub import {
my $self = shift;
@@ -32,51 +33,44 @@
}
-my $net_timeout; # max seconds to wait for a response from the added content vendor
+my $handler; # added content handler class handle
my $cache; # memcache handle
+my $net_timeout; # max seconds to wait for a response from the added content vendor
my $max_errors; # max consecutive lookup failures before added content is temporarily disabled
my $error_countdown; # current consecutive errors countdown
-my $jacket_url; # URL for fetching jacket images
# number of seconds to wait before next lookup
# is attempted after lookups have been disabled
my $error_retry_timeout;
-my $init = 0; # has the child init been run?
sub child_init {
OpenSRF::System->bootstrap_client( config_file => $bs_config );
+ $cache = OpenSRF::Utils::Cache->new;
my $sclient = OpenSRF::Utils::SettingsClient->new();
my $ac_data = $sclient->config_value("added_content");
return unless $ac_data;
+ my $ac_handler = $ac_data->{module};
+ return unless $ac_handler;
- $cache = OpenSRF::Utils::Cache->new;
-
$net_timeout = $ac_data->{timeout} || 1;
$error_countdown = $max_errors = $ac_data->{max_errors} || 10;
- $jacket_url = $ac_data->{jacket_url};
$error_retry_timeout = $ac_data->{retry_timeout} || 600;
- $init = 1;
-
- my $ac_handler = $ac_data->{module};
+ $logger->debug("Attempting to load Added Content handler: $ac_handler");
- if($ac_handler) {
- $logger->debug("Attempting to load Added Content handler: $ac_handler");
-
- eval "use $ac_handler";
-
- if($@) {
- $logger->error("Unable to load Added Content handler [$ac_handler]: $@");
- return;
- }
-
- $handler = $ac_handler->new($ac_data);
- $logger->debug("added content loaded handler: $handler");
+ eval "use $ac_handler";
+
+ if($@) {
+ $logger->error("Unable to load Added Content handler [$ac_handler]: $@");
+ return;
}
+
+ $handler = $ac_handler->new($ac_data);
+ $logger->debug("added content loaded handler: $handler");
}
@@ -85,54 +79,81 @@
my $r = shift;
my $cgi = CGI->new;
my $path = $r->path_info;
+ my $res;
- my( undef, $data, $format, $key ) = split(/\//, $r->path_info);
- return Apache2::Const::NOT_FOUND unless $data and $format and $key;
+ my( undef, $type, $format, $key ) = split(/\//, $r->path_info);
- child_init() unless $init;
- return Apache2::Const::NOT_FOUND unless $init;
+ child_init() unless $handler;
- return fetch_jacket($format, $key) if $data eq 'jacket';
- return Apache2::Const::NOT_FOUND unless lookups_enabled();
+ return Apache2::Const::NOT_FOUND unless $handler and $type and $format and $key;
+ return $res if defined($res = $AC->serve_from_cache($type, $format, $key));
+ return Apache2::Const::NOT_FOUND unless $AC->lookups_enabled;
my $err;
- my $success;
- my $method = "${data}_${format}";
+ my $data;
+ my $method = "${type}_${format}";
+ return Apache2::Const::NOT_FOUND unless $handler->can($method);
+
try {
- $success = $handler->$method($key);
- } catch Error with {
- my $err = shift;
- $logger->error("added content handler failed: $method($key) => $err");
+ $data = $handler->$method($key);
+ } catch Error with {
+ $err = shift;
decr_error_countdown();
+ $logger->error("added content handler failed: $method($key) => $err");
};
- return Apache2::Const::NOT_FOUND if $err or !$success;
+ return Apache2::Const::NOT_FOUND if $err;
+
+ if(!$data) {
+ # if the AC lookup found no corresponding data, cache that information
+ $logger->debug("added content handler returned no results $method($key)") unless $data;
+ $AC->cache_result($type, $format, $key, {nocontent=>1});
+ return Apache2::Const::NOT_FOUND;
+ }
+
+ $AC->print_content($data);
+ $AC->cache_result($type, $format, $key, $data);
+
+ reset_error_countdown();
return Apache2::Const::OK;
}
-sub decr_error_countdown {
- $error_countdown--;
- if($error_countdown < 1) {
- $logger->warn("added content error count exhausted. Disabling lookups for $error_retry_timeout seconds");
- disable_lookups();
+sub print_content {
+ my($class, $data, $from_cache) = @_;
+ return Apache2::Const::NOT_FOUND if $data->{nocontent};
+
+ my $ct = $data->{content_type};
+ my $content = $data->{content};
+ print "Content-type: $ct\n\n";
+
+ if($data->{binary}) {
+ binmode STDOUT;
+ # if it hasn't been cached yet, it's still in binary form
+ print( ($from_cache) ? decode_base64($content) : $content );
+ } else {
+ print $content;
}
-}
-sub reset_error_countdown {
- $error_countdown = $max_errors;
+
+ return Apache2::Const::OK;
}
-# generic GET call
+
+
+# returns an HTPP::Response object
sub get_url {
my( $self, $url ) = @_;
- $logger->info("added content getting [timeout=$net_timeout] URL = $url");
+
+ $logger->info("added content getting [timeout=$net_timeout, errors_remaining=$error_countdown] URL = $url");
my $agent = LWP::UserAgent->new(timeout => $net_timeout);
- my $res = $agent->get($url);
+
+ my $res = $agent->get($url);
+ $logger->info("added content request returned with code " . $res->code);
die "added content request failed: " . $res->status_line ."\n" unless $res->is_success;
- reset_error_countdown();
- return $res->content;
+
+ return $res;
}
sub lookups_enabled {
@@ -147,82 +168,33 @@
$cache->put_cache('ac.no_lookup', 1, $error_retry_timeout);
}
-sub fetch_jacket {
- my($size, $isbn) = @_;
- return Apache2::Const::NOT_FOUND unless $jacket_url and $size and $isbn;
-
- if($size eq 'small') {
-
- # try to serve small images from the cache first
- my $img_data = $cache->get_cache("ac.$size.$isbn");
-
- if($img_data) {
-
- $logger->debug("serving jacket $isbn from cache...");
-
- my $c_type = $img_data->{content_type};
- my $img = decode_base64($img_data->{img});
-
- print "Content-type: $c_type\n\n";
-
- binmode STDOUT;
- print $img;
- return Apache2::Const::OK;
- }
+sub decr_error_countdown {
+ $error_countdown--;
+ if($error_countdown < 1) {
+ $logger->warn("added content error count exhausted. Disabling lookups for $error_retry_timeout seconds");
+ $AC->disable_lookups;
}
+}
- if(!lookups_enabled()) {
- $error_countdown = $max_errors; # reset the counter
- return Apache2::Const::NOT_FOUND;
- }
+sub reset_error_countdown {
+ $error_countdown = $max_errors;
+}
- (my $url = $jacket_url) =~ s/\${isbn}/$isbn/ig;
+sub cache_result {
+ my($class, $type, $format, $key, $data) = @_;
+ $logger->debug("caching $type/$format/$key");
+ $data->{content} = encode_base64($data->{content}) if $data->{binary};
+ return $cache->put_cache("ac.$type.$format.$key", $data);
+}
- $logger->debug("added content getting jacket with timeout=$net_timeout and URL = $url");
-
- my $res;
- my $err;
-
- try {
- my $agent = LWP::UserAgent->new(timeout => $net_timeout);
- $res = $agent->get($url);
- } catch Error with {
- $err = shift;
- $logger->error("added content lookup died with $err");
- };
-
- if( $err or $res->code == 500 ) {
- $logger->warn("added content jacket fetch failed (retries remaining = $error_countdown) " .
- (($res) ? $res->status_line : "$err"));
- decr_error_countdown();
- return Apache2::Const::NOT_FOUND;
- }
-
- return Apache2::Const::NOT_FOUND unless $res->code == 200;
-
- # ignore old errors after a successful lookup
- reset_error_countdown();
-
- my $c_type = $res->header('Content-type');
- my $binary_img = $res->content;
- print "Content-type: $c_type\n\n";
-
- binmode STDOUT;
- print $binary_img;
-
- $cache->put_cache(
- "ac.$size.$isbn", {
- content_type => $c_type,
- img => encode_base64($binary_img,'')
- }
- ) if $size eq 'small';
-
- return Apache2::Const::OK;
+sub serve_from_cache {
+ my($class, $type, $format, $key) = @_;
+ my $data = $cache->get_cache("ac.$type.$format.$key");
+ return undef unless $data;
+ $logger->debug("serving $type/$format/$key from cache");
+ return $class->print_content($data, 1);
}
-
-
1;
-
More information about the open-ils-commits
mailing list