[open-ils-commits] r210 - in servres/trunk/conifer: . libsystems libsystems/evergreen libsystems/z3950 syrup templates (gfawcett)
svn at svn.open-ils.org
svn at svn.open-ils.org
Sun Mar 22 21:06:09 EDT 2009
Author: gfawcett
Date: 2009-03-22 21:06:07 -0400 (Sun, 22 Mar 2009)
New Revision: 210
Added:
servres/trunk/conifer/libsystems/
servres/trunk/conifer/libsystems/__init__.py
servres/trunk/conifer/libsystems/evergreen/
servres/trunk/conifer/libsystems/evergreen/__init__.py
servres/trunk/conifer/libsystems/evergreen/item_status.py
servres/trunk/conifer/libsystems/evergreen/support.py
servres/trunk/conifer/libsystems/z3950/
servres/trunk/conifer/libsystems/z3950/__init__.py
servres/trunk/conifer/libsystems/z3950/marctools.py
servres/trunk/conifer/libsystems/z3950/yaz_search.py
servres/trunk/conifer/templates/graham_z3950_test.xhtml
Modified:
servres/trunk/conifer/syrup/urls.py
servres/trunk/conifer/syrup/views.py
Log:
Z39.50 + Evergreen demo: are items available, holdable?
see /syrup/graham_z3950test/ . I am using Evergreen in this example,
but it could be replaced by SIP or another backend that can take a bib
ID and return availability information.
(If SIP needs a barcode, not bib ID (which I suspect it might), then
we may need another lookup function in the interface. Will investigate.
Added: servres/trunk/conifer/libsystems/__init__.py
===================================================================
Added: servres/trunk/conifer/libsystems/evergreen/__init__.py
===================================================================
Added: servres/trunk/conifer/libsystems/evergreen/item_status.py
===================================================================
--- servres/trunk/conifer/libsystems/evergreen/item_status.py (rev 0)
+++ servres/trunk/conifer/libsystems/evergreen/item_status.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -0,0 +1,26 @@
+import warnings
+from support import ER, E1
+from pprint import pprint
+
+# Proposing this as an interface method. Given a bib ID, return a dict
+# giving the item's bibid, barcode, availability (boolean),
+# holdability (boolean), and location (a string description). If the
+# bib ID is invalid, return None.
+
+def lookup_availability(bib_id):
+ rec = E1('open-ils.search.asset.copy.fleshed2.retrieve', bib_id)
+ if 'stacktrace' in rec:
+ warnings.warn(repr(('no such bib id', bib_id, repr(rec))))
+ return None
+ resp = {
+ 'bibid': bib_id,
+ 'barcode': rec['barcode'],
+ 'available': rec['status']['name'] == 'Available',
+ 'holdable': rec['status']['holdable'] == 't',
+ 'location': rec['location']['name']}
+ return resp
+
+
+if __name__ == '__main__':
+ DYLAN = 1321798
+ print lookup_availability(DYLAN)
Added: servres/trunk/conifer/libsystems/evergreen/support.py
===================================================================
--- servres/trunk/conifer/libsystems/evergreen/support.py (rev 0)
+++ servres/trunk/conifer/libsystems/evergreen/support.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -0,0 +1,83 @@
+import warnings
+import urllib2
+from urllib import quote
+import simplejson as json
+from xml.etree import ElementTree
+import re
+import sys
+
+#------------------------------------------------------------
+# Configuration
+
+# where is our evergreen server's opensrf http gateway?
+
+BASE = 'http://dwarf.cs.uoguelph.ca/osrf-gateway-v1'
+LOCALE = 'en-US'
+
+# where can I find a copy of fm_IDL.xml from Evergreen?
+
+# # This will work always, though maybe you want to up the rev number...
+# FM_IDL_LOCATION = ('http://svn.open-ils.org/trac/ILS/export/12640'
+# '/trunk/Open-ILS/examples/fm_IDL.xml')
+
+# # or, if you have a local copy...
+# FM_IDL_LOCATION = 'file:fm_IDL.xml'
+
+FM_IDL_LOCATION = 'http://dwarf.cs.uoguelph.ca/reports/fm_IDL.xml'
+
+#------------------------------------------------------------
+# parse fm_IDL, to build a field-name-lookup service.
+
+def _fields():
+ tree = ElementTree.parse(urllib2.urlopen(FM_IDL_LOCATION))
+ NS = '{http://opensrf.org/spec/IDL/base/v1}'
+ for c in tree.findall('%sclass' % NS):
+ cid = c.attrib['id']
+ fields = [f.attrib['name'] \
+ for f in c.findall('%sfields/%sfield' % (NS,NS))]
+ yield (cid, fields)
+
+fields_for_class = dict(_fields())
+
+#------------------------------------------------------------
+
+def evergreen_object(rec):
+ """Where possible, add field-names to an Evergreen return-value."""
+ if isinstance(rec, list):
+ return map(evergreen_object, rec)
+ if not (isinstance(rec, dict) and '__c' in rec):
+ return rec
+ else:
+ kls = rec['__c']
+ data = rec['__p']
+ fields = fields_for_class[kls]
+ #print '----', (kls, fields)
+ return dict(zip(fields, map(evergreen_object, data)))
+
+def evergreen_request(method, *args, **kwargs):
+ service = '.'.join(method.split('.')[:2])
+ kwargs.setdefault('locale', LOCALE)
+ kwargs.update({'service':service, 'method':method})
+ params = ['%s=%s' % (k,quote(v)) for k,v in kwargs.items()]
+ params += ['param=%s' % quote(str(a)) for a in args]
+ url = '%s?%s' % (BASE, '&'.join(params))
+ req = urllib2.urlopen(url)
+ resp = json.load(req)
+ assert resp['status'] == 200, 'error during evergreen request'
+ payload = resp['payload']
+ #print '----', payload
+ return evergreen_object(payload)
+
+def evergreen_request_single_result(method, *args):
+ resp = evergreen_request(method, *args)
+ if len(resp) > 1:
+ warnings.warn('taking single value from multivalue evergreen response')
+ print >> sys.stderr, repr(resp)
+ return resp[0]
+
+
+#------------------------------------------------------------
+# Abbreviations
+
+ER = evergreen_request
+E1 = evergreen_request_single_result
Added: servres/trunk/conifer/libsystems/z3950/__init__.py
===================================================================
Added: servres/trunk/conifer/libsystems/z3950/marctools.py
===================================================================
--- servres/trunk/conifer/libsystems/z3950/marctools.py (rev 0)
+++ servres/trunk/conifer/libsystems/z3950/marctools.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -0,0 +1,112 @@
+"""
+ MARC utlities
+ Public Domain 2007 public.resource.org
+
+ Author: Joel Hardi <joel at hardi.org>
+"""
+
+class locToUTF8(object):
+ "Changes text from LOC into unicode, using replace() method"
+
+ dict = {}
+ charmap = {}
+
+ def __init__(self):
+ "Sets self.dict and search character index self.charmap"
+ self.dict = {
+ "\X20":"\u0020", # "HARD SPACE - represented by a space"
+ "\XC2\XA1":"\u00A1", # "INVERTED EXCLAMATION MARK"
+ "\XC2\XA3":"\u00A3", # "BRITISH POUND / POUND SIGN"
+ "\XC2\XA9":"\u00A9", # "COPYRIGHT SIGN"
+ "\XC2\XAE":"\u00AE", # "PATENT MARK / REGISTERED SIGN"
+ "\XC2\XB0":"\u00B0", # "DEGREE SIGN"
+ "\XC2\XB1":"\u00B1", # "PLUS OR MINUS / PLUS-MINUS SIGN"
+ "\XC2\XB7":"\u00B7", # "MIDDLE DOT"
+ "\XC2\XBF":"\u00BF", # "INVERTED QUESTION MARK"
+ "\XC3\X86":"\u00C6", # "UPPERCASE DIGRAPH AE / LATIN CAPITAL LIGATURE AE"
+ "\XC3\X98":"\u00D8", # "UPPERCASE SCANDINAVIAN O / LATIN CAPITAL LETTER O WITH STROKE"
+ "\XC3\X9E":"\u00DE", # "UPPERCASE ICELANDIC THORN / LATIN CAPITAL LETTER THORN (Icelandic)"
+ "\XC3\XA6":"\u00E6", # "LOWERCASE DIGRAPH AE / LATIN SMALL LIGATURE AE"
+ "\XC3\XB0":"\u00F0", # "LOWERCASE ETH / LATIN SMALL LETTER ETH (Icelandic)"
+ "\XC3\XB8":"\u00F8", # "LOWERCASE SCANDINAVIAN O / LATIN SMALL LETTER O WITH STROKE"
+ "\XC3\XBE":"\u00FE", # "LOWERCASE ICELANDIC THORN / LATIN SMALL LETTER THORN (Icelandic)"
+ "\XC4\X90":"\u0110", # "UPPERCASE D WITH CROSSBAR / LATIN CAPITAL LETTER D WITH STROKE"
+ "\XC4\X91":"\u0111", # "LOWERCASE D WITH CROSSBAR / LATIN SMALL LETTER D WITH STROKE"
+ "\XC4\XB1":"\u0131", # "LOWERCASE TURKISH I / LATIN SMALL LETTER DOTLESS I"
+ "\XC5\X81":"\u0141", # "UPPERCASE POLISH L / LATIN CAPITAL LETTER L WITH STROKE"
+ "\XC5\X82":"\u0142", # "LOWERCASE POLISH L / LATIN SMALL LETTER L WITH STROKE"
+ "\XC5\X92":"\u0152", # "UPPERCASE DIGRAPH OE / LATIN CAPITAL LIGATURE OE"
+ "\XC5\X93":"\u0153", # "LOWERCASE DIGRAPH OE / LATIN SMALL LIGATURE OE"
+ "\XC6\XA0":"\u01A0", # "UPPERCASE O-HOOK / LATIN CAPITAL LETTER O WITH HORN"
+ "\XC6\XA1":"\u01A1", # "LOWERCASE O-HOOK / LATIN SMALL LETTER O WITH HORN"
+ "\XC6\XAF":"\u01AF", # "UPPERCASE U-HOOK / LATIN CAPITAL LETTER U WITH HORN"
+ "\XC6\XB0":"\u01B0", # "LOWERCASE U-HOOK / LATIN SMALL LETTER U WITH HORN"
+ "\XCA\XB9":"\u02B9", # "SOFT SIGN, PRIME / MODIFIER LETTER PRIME"
+ "\XCA\XBA":"\u02BA", # "HARD SIGN, DOUBLE PRIME / MODIFIER LETTER DOUBLE PRIME"
+ "\XCA\XBB":"\u02BB", # "AYN / MODIFIER LETTER TURNED COMMA"
+ "\XCA\XBE":"\u02BE", # "ALIF / MODIFIER LETTER RIGHT HALF RING"
+ "\XCC\X80":"\u0300", # "GRAVE / COMBINING GRAVE ACCENT (Varia)"
+ "\XCC\X81":"\u0301", # "ACUTE / COMBINING ACUTE ACCENT (Oxia)"
+ "\XCC\X82":"\u0302", # "CIRCUMFLEX / COMBINING CIRCUMFLEX ACCENT"
+ "\XCC\X83":"\u0303", # "TILDE / COMBINING TILDE"
+ "\XCC\X84":"\u0304", # "MACRON / COMBINING MACRON"
+ "\XCC\X86":"\u0306", # "BREVE / COMBINING BREVE (Vrachy)"
+ "\XCC\X87":"\u0307", # "SUPERIOR DOT / COMBINING DOT ABOVE"
+ "\XCC\X88":"\u0308", # "UMLAUT, DIAERESIS / COMBINING DIAERESIS (Dialytika)"
+ "\XCC\X89":"\u0309", # "PSEUDO QUESTION MARK / COMBINING HOOK ABOVE"
+ "\XCC\X8A":"\u030A", # "CIRCLE ABOVE, ANGSTROM / COMBINING RING ABOVE"
+ "\XCC\X8B":"\u030B", # "DOUBLE ACUTE / COMBINING DOUBLE ACUTE ACCENT"
+ "\XCC\X8C":"\u030C", # "HACEK / COMBINING CARON"
+ "\XCC\X90":"\u0310", # "CANDRABINDU / COMBINING CANDRABINDU"
+ "\XCC\X93":"\u0313", # "HIGH COMMA, CENTERED / COMBINING COMMA ABOVE (Psili)"
+ "\XCC\X95":"\u0315", # "HIGH COMMA, OFF CENTER / COMBINING COMMA ABOVE RIGHT"
+ "\XCC\X9C":"\u031C", # "RIGHT CEDILLA / COMBINING LEFT HALF RING BELOW"
+ "\XCC\XA3":"\u0323", # "DOT BELOW / COMBINING DOT BELOW"
+ "\XCC\XA4":"\u0324", # "DOUBLE DOT BELOW / COMBINING DIAERESIS BELOW"
+ "\XCC\XA5":"\u0325", # "CIRCLE BELOW / COMBINING RING BELOW"
+ "\XCC\XA6":"\u0326", # "LEFT HOOK (COMMA BELOW) / COMBINING COMMA BELOW"
+ "\XCC\XA7":"\u0327", # "CEDILLA / COMBINING CEDILLA"
+ "\XCC\XA8":"\u0328", # "RIGHT HOOK, OGONEK / COMBINING OGONEK"
+ "\XCC\XAE":"\u032E", # "UPADHMANIYA / COMBINING BREVE BELOW"
+ "\XCC\XB2":"\u0332", # "UNDERSCORE / COMBINING LOW LINE"
+ "\XCC\XB3":"\u0333", # "DOUBLE UNDERSCORE / COMBINING DOUBLE LOW LINE"
+ "\XE2\X84\X93":"\u2113", # "SCRIPT SMALL L"
+ "\XE2\X84\X97":"\u2117", # "SOUND RECORDING COPYRIGHT"
+ "\XE2\X99\XAD":"\u266D", # "MUSIC FLAT SIGN"
+ "\XE2\X99\XAF":"\u266F", # "MUSIC SHARP SIGN"
+ "\XEF\XB8\XA0":"\uFE20", # "LIGATURE, FIRST HALF / COMBINING LIGATURE LEFT HALF"
+ "\XEF\XB8\XA1":"\uFE21", # "LIGATURE, SECOND HALF / COMBINING LIGATURE RIGHT HALF"
+ "\XEF\XB8\XA2":"\uFE22", # "DOUBLE TILDE, FIRST HALF / COMBINING DOUBLE TILDE LEFT HALF"
+ "\XEF\XB8\XA3":"\uFE23", # "DOUBLE TILDE, SECOND HALF / COMBINING DOUBLE TILDE RIGHT HALF"
+ }
+
+ # build self.charmap to map each first char of a search string to a list of its search strings
+ firstchars = []
+ self.charmap = {}
+ for i in self.dict.iterkeys():
+ if firstchars.count(i[0]) == 0:
+ firstchars.append(i[0])
+ self.charmap[i[0]] = []
+ self.charmap[i[0]].append(i)
+
+ def replace(self, str):
+ "Given string str, returns unicode string with correct character replcements"
+ searchchars = []
+ # build subset of search/replace pairs to use based on if first char of search appears in str
+ prev = range(0,3)
+ for c in str:
+ prev[0] = prev[1]
+ prev[1] = prev[2]
+ prev[2] = c
+ if self.charmap.has_key(c):
+ if searchchars.count(c) == 0:
+ searchchars.append(c)
+ elif ord(c) > 127 and prev.count(c) == 0:
+ str = str.replace(c, '\\X%x' % ord(c))
+
+ # perform search/replaces
+ for c in searchchars:
+ for i in self.charmap[c]:
+ str = str.replace(i, self.dict[i])
+
+ return unicode(str, 'raw-unicode-escape')
Added: servres/trunk/conifer/libsystems/z3950/yaz_search.py
===================================================================
--- servres/trunk/conifer/libsystems/z3950/yaz_search.py (rev 0)
+++ servres/trunk/conifer/libsystems/z3950/yaz_search.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -0,0 +1,85 @@
+# z39.50 search using yaz-client.
+# dependencies: yaz-client, pexpect
+
+# I found that pyz3950.zoom seemed wonky when testing against conifer
+# z3950, so I whipped up this expect-based version instead.
+
+import warnings
+import re
+from xml.etree import ElementTree
+import pexpect
+import marctools
+loc_to_unicode = marctools.locToUTF8().replace
+
+LOG = None # for pexpect debugging, try LOG = sys.stderr
+YAZ_CLIENT = 'yaz-client'
+GENERAL_TIMEOUT = 3
+PRESENT_TIMEOUT = 30
+
+
+def search(host, database, query, start=1, limit=None):
+
+ server = pexpect.spawn('yaz-client', timeout=GENERAL_TIMEOUT, logfile=LOG)
+ for line in ('open %s' % host, 'base %s' % database, 'format xml'):
+ server.sendline(line)
+ server.expect('Z>')
+
+ # send the query
+ # note, we're using prefix queries for the moment.
+ server.sendline('find %s' % query)
+ server.expect(r'Number of hits: (\d+).*')
+ numhits = int(server.match.group(1))
+ if start > numhits:
+ warnings.warn('asked z3950 to start at %d, but only %d results.' % (start, numhits))
+ return []
+
+ # how many to present? At most 10 for now.
+ to_show = min(numhits-1, 10) # minus 1 for dwarf ??
+ if limit:
+ to_show = min(to_show, limit)
+ server.expect('Z>')
+ server.sendline('show %s + %d' % (start, to_show))
+ err = server.expect_list([re.compile(r'Records: (\d+)'),
+ re.compile('Target closed connection')])
+ if err:
+ warnings.warn('error during z3950 conversation.')
+ server.close()
+ return []
+
+ raw_records = []
+ for x in range(to_show):
+ server.expect(r'Record type: XML', timeout=PRESENT_TIMEOUT)
+ server.expect('<record .*</record>')
+ raw_records.append(server.match.group(0))
+
+ server.expect('nextResultSetPosition')
+ server.expect('Z>')
+ server.sendline('quit')
+ server.close()
+
+ parsed = []
+ for rec in raw_records:
+ dct = {}
+ parsed.append(dct)
+ tree = ElementTree.fromstring(rec)
+ for df in tree.findall('{http://www.loc.gov/MARC21/slim}datafield'):
+ t = df.attrib['tag']
+ for sf in df.findall('{http://www.loc.gov/MARC21/slim}subfield'):
+ c = sf.attrib['code']
+ v = sf.text
+ dct[t+c] = loc_to_unicode(v)
+
+ return parsed
+
+#------------------------------------------------------------
+# some tests
+
+if __name__ == '__main__':
+ print loc_to_unicode('A\\XCC\\X81n')
+ tests = [
+ ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Musson" "Evil"'),
+ ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Denis" "Gravel"'),
+ ('z3950.loc.gov:7090', 'VOYAGER', '@attr 1=4 @attr 4=1 "dylan"')]
+ for host, db, query in tests:
+ print (host, db, query)
+ print search(host, db, query, limit=1)
Modified: servres/trunk/conifer/syrup/urls.py
===================================================================
--- servres/trunk/conifer/syrup/urls.py 2009-03-20 02:52:45 UTC (rev 209)
+++ servres/trunk/conifer/syrup/urls.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -16,6 +16,7 @@
(r'^browse/(?P<browse_option>.*)/$', 'browse'),
(r'^prefs/$', 'user_prefs'),
(r'^z3950test/$', 'z3950_test'),
+ (r'^graham_z3950test/$', 'graham_z3950_test'),
#MARK: propose we kill open_courses, we have browse.
(r'^opencourse/$', 'open_courses'),
(r'^search/$', 'search'),
Modified: servres/trunk/conifer/syrup/views.py
===================================================================
--- servres/trunk/conifer/syrup/views.py 2009-03-20 02:52:45 UTC (rev 209)
+++ servres/trunk/conifer/syrup/views.py 2009-03-23 01:06:07 UTC (rev 210)
@@ -241,9 +241,22 @@
# print("done searching...")
res_str = "" . join(collector)
# print(res_str)
-
return g.render('z3950_test.xhtml', res_str=res_str)
+def graham_z3950_test(request):
+ from conifer.libsystems.z3950 import yaz_search
+ from conifer.libsystems.evergreen.item_status import lookup_availability
+ host, db, query = ('dwarf.cs.uoguelph.ca:2210', 'conifer', '@and "Denis" "Gravel"')
+ #host, db, query = ('z3950.loc.gov:7090', 'VOYAGER', '@attr 1=4 @attr 4=1 "dylan"')
+ results = yaz_search.search(host, db, query)
+ for result in results:
+ bibid = result.get('901c')
+ if bibid:
+ avail = lookup_availability(bibid)
+ if avail:
+ result['avail'] = avail
+ return g.render('graham_z3950_test.xhtml', results=results)
+
def browse(request, browse_option=''):
#the defaults should be moved into a config file or something...
page_num = int(request.GET.get('page', 1))
Added: servres/trunk/conifer/templates/graham_z3950_test.xhtml
===================================================================
--- servres/trunk/conifer/templates/graham_z3950_test.xhtml (rev 0)
+++ servres/trunk/conifer/templates/graham_z3950_test.xhtml 2009-03-23 01:06:07 UTC (rev 210)
@@ -0,0 +1,48 @@
+<?python
+title = _('Z39.50 Test, with Evergreen giving item availability')
+# I just made up these keys...
+keys = [('245a', 'Title'), ('100a', 'Author'), ('260c', 'PubDate'), ('901c', 'BibID')]
+?>
+<html xmlns="http://www.w3.org/1999/xhtml"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ xmlns:py="http://genshi.edgewall.org/">
+<xi:include href="master.xhtml"/>
+<xi:include href="paginate.xhtml"/>
+<head>
+ <title>${title}</title>
+ <style>
+ tbody.available { background-color: #dfd; }
+ .lesser { font-size: smaller; color: gray; display: none; }
+ </style>
+ <script type="text/javascript">
+ <!-- !This ought to be in paginate.xhtml, not here. how to do? -->
+ $(function() { $('.pagetable').tablesorter(); });
+ </script>
+</head>
+<body>
+ <h1>${title}</h1>
+ <p><a href="javascript:$('.lesser').show(); void(0);">show more detail</a></p>
+ <table py:for="res in results" style="margin: 1em 0; border: black 1px solid;">
+ <?python
+ is_available = res.get('avail') and (res['avail']['available'] or res['avail']['holdable'])
+ ?>
+ <tbody class="${is_available and 'available' or ''}">
+ <tr py:for="k, title in keys" py:if="k in res">
+ <th>${title}</th><td>${res[k]}</td>
+ </tr>
+ <tr py:if="'avail' in res">
+ <th>Availability</th>
+ <td>
+ <div py:for="k,v in res['avail'].items()">
+ ${'%s = %s' % (k,v)}
+ </div>
+ </td>
+ </tr>
+ <?python allkeys = res.keys(); allkeys.sort(); ?>
+ <tr py:for="k in allkeys" class="lesser">
+ <th>${k}</th><td>${res[k]}</td>
+ </tr>
+ </tbody>
+ </table>
+</body>
+</html>
More information about the open-ils-commits
mailing list