[open-ils-commits] [GIT] Evergreen ILS branch master updated. ad10cde090791e02cb106e36a310581bfaafface
Evergreen Git
git at git.evergreen-ils.org
Tue Aug 9 12:42:37 EDT 2016
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via ad10cde090791e02cb106e36a310581bfaafface (commit)
via 10aaac3955bd6baaa68f502c1ef851fce093b183 (commit)
via 42d5c82b1ca74ca4aa0581b930ecf0b22772fefe (commit)
via 85a470ef7144265ee0a2618fafffeac8bb36b65d (commit)
from 9bdfdf9a870dceb5baa92ddde6575b9a34a624fb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit ad10cde090791e02cb106e36a310581bfaafface
Author: Mike Rylander <mrylander at gmail.com>
Date: Tue Aug 9 12:41:54 2016 -0400
Stamping upgrade for accent-ignoring patron search
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 27f5d85..8e08b18 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0985', :eg_version); -- dbwells/gmcharlt/miker
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0986', :eg_version); -- dpearl/dyrcona/miker
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql b/Open-ILS/src/sql/Pg/upgrade/0986.schema.patron_unaccent.sql
similarity index 95%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql
rename to Open-ILS/src/sql/Pg/upgrade/0986.schema.patron_unaccent.sql
index 3ec260b..7fb0465 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0986.schema.patron_unaccent.sql
@@ -1,6 +1,6 @@
BEGIN;
-SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('0986', :eg_version);
CREATE EXTENSION IF NOT EXISTS unaccent SCHEMA public;
commit 10aaac3955bd6baaa68f502c1ef851fce093b183
Author: Jason Stephenson <jstephenson at mvlcstaff.org>
Date: Thu Mar 17 17:47:52 2016 -0400
LP 1501781: Make patron name search diacritic/space insensitive.
Add pgtap tests for the new org unit setting, the new indexes, and to
test the evergreen.unaccent_and_squash() functionality. Many of the
latter tests are based on the tests for evergreen.lowercase().
Signed-off-by: Jason Stephenson <jason at sigio.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/t/lp1501781-unaccent_and_squash.pg b/Open-ILS/src/sql/Pg/t/lp1501781-unaccent_and_squash.pg
new file mode 100644
index 0000000..f09dcf6
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/t/lp1501781-unaccent_and_squash.pg
@@ -0,0 +1,61 @@
+-- Start transaction and plan the tests.
+BEGIN;
+
+-- Number of tests that will be run
+SELECT plan(15);
+
+-- Run the tests
+
+SELECT isnt_empty(
+ 'SELECT * FROM config.org_unit_setting_type WHERE name = $$circ.patron_search.diacritic_insensitive$$',
+ 'circ.patron_search.diacritic_insensitive exists');
+
+SELECT has_index('actor', 'usr', 'actor_usr_first_given_name_unaccent_idx',
+ 'unaccent_and_squash(first_given_name)',
+ 'Index actor_usr_first_given_name_unaccent_idx exists');
+
+SELECT has_index('actor', 'usr', 'actor_usr_second_given_name_unaccent_idx',
+ 'unaccent_and_squash(second_given_name)',
+ 'Index actor_usr_second_given_name_unaccent_idx exists');
+
+SELECT has_index('actor', 'usr', 'actor_usr_family_name_unaccent_idx',
+ 'unaccent_and_squash(family_name)',
+ 'Index actor_usr_family_name_unaccent_idx exists');
+
+SELECT is(evergreen.unaccent_and_squash('aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ'),
+ 'aabbccddeeffgghhiijjkkllmmnnooppqqrrssttuuvvwwxxyyzz',
+ 'English Characters');
+
+SELECT is(evergreen.unaccent_and_squash('0123456789!@#$%^&*()-+'),
+ '0123456789!@#$%^&*()-+', 'Numbers and Symbols');
+
+SELECT is(evergreen.unaccent_and_squash('ÁÉÍÑÓÚÜáéíñóúü¿¡'),
+ 'aeinouuaeinouu¿¡', 'Spanish');
+
+SELECT is(evergreen.unaccent_and_squash('ÖÜÇĞŞöüçğşİI'),
+ 'oucgsoucgsii', 'Turkish');
+
+SELECT is(evergreen.unaccent_and_squash('ÄÉÖÜ'),
+ 'aeou', 'German');
+
+SELECT is(evergreen.unaccent_and_squash('АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
+ 'абвгдежзийклмнопрстуфхцчшщъыьэюя', 'Russian');
+
+SELECT is(evergreen.unaccent_and_squash('ÁĄÄÉĘĚÍÓÔÚŮÝČĹŇŔŘŠŽ'),
+ 'aaaeeeioouuyclnrrsz', 'Czeck, Slovack & Slovenian');
+
+SELECT is(evergreen.unaccent_and_squash('Άά Έέ Ήή Ίί Όό Ύύ Ώώ'),
+ 'άάέέήήίίόόύύώώ', 'Greek');
+
+SELECT is(evergreen.unaccent_and_squash('Jacques Bélanger'),
+ 'jacquesbelanger', 'French nom and prénom');
+
+SELECT is(evergreen.unaccent_and_squash('El Niño'),
+ 'elnino', 'Spanish "The Kid"');
+
+SELECT is(evergreen.unaccent_and_squash('Œuvres'),
+ 'euvres', 'oe ligature');
+
+-- Finish the test and clean up.
+SELECT * FROM finish();
+ROLLBACK;
commit 42d5c82b1ca74ca4aa0581b930ecf0b22772fefe
Author: Jason Stephenson <jason at sigio.com>
Date: Thu Mar 17 08:01:54 2016 -0400
LP 1501781: Make patron name search diacritic/space insensitive.
This commit adds the necessary code to storage actor publisher to use
the new evergreen.unaccent_and_squash db function when doing a patron
name search.
It is based on code originally by Dan Pearl, but then modified to
remove a dependency on a new Perl module by calling the db function on
the search input as well.
Signed-off-by: Jason Stephenson <jason at sigio.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/actor.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/actor.pm
index e2a4edc..06ba202 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/actor.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/actor.pm
@@ -5,12 +5,16 @@ use OpenSRF::Utils::Logger qw/:level/;
use OpenSRF::Utils qw/:datetime/;
use OpenILS::Utils::Fieldmapper;
use OpenSRF::Utils::SettingsClient;
-
+use OpenILS::Application::AppUtils;
+use OpenSRF::Utils::JSON;
use DateTime;
use DateTime::Format::ISO8601;
use DateTime::Set;
use DateTime::SpanSet;
+my $U = "OpenILS::Application::AppUtils";
+my $JSON = "OpenSRF::Utils::JSON";
+
my $_dt_parser = DateTime::Format::ISO8601->new;
my $log = 'OpenSRF::Utils::Logger';
@@ -617,6 +621,15 @@ __PACKAGE__->register_method(
NOTE
);
+sub _prepare_name_argument {
+ # Get rid of extra spaces, accents, and regex characters
+ my ($search) = _clean_regex_chars(@_);
+ my $sth = actor::user->db_Main->prepare_cached("SELECT evergreen.unaccent_and_squash(?)");
+ $sth->execute($search);
+ my $r = $sth->fetch;
+ return ($r && @$r) ? $r->[0] : $search;
+};
+
sub _clean_regex_chars {
my ($search) = @_;
@@ -664,8 +677,23 @@ sub patron_search {
# group 2 = phone, ident
# group 3 = barcode
- my $usr = join ' AND ', map { "evergreen.lowercase(CAST($_ AS text)) ~ ?" } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
- my @usrv = map { "^" . _clean_regex_chars($$search{$_}{value}) } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
+ # Treatment of name fields depends on whether the org has
+ # diacritic_insensitivity turned on or off.
+
+ my $diacritic_insensitive = $U->ou_ancestor_setting_value($ws_ou, 'circ.patron_search.diacritic_insensitive');
+ # Parse from JSON to Perl boolean (1|0):
+ $diacritic_insensitive = ($diacritic_insensitive) ? $JSON->JSON2perl($diacritic_insensitive) : 0;
+ my $usr;
+ my @usrv;
+
+ if ($diacritic_insensitive) {
+ $usr = join ' AND ', map { "evergreen.unaccent_and_squash(CAST($_ AS text)) ~ ?" } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
+ @usrv = map { "^" . _prepare_name_argument($$search{$_}{value}) } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
+
+ } else {
+ $usr = join ' AND ', map { "evergreen.lowercase(CAST($_ AS text)) ~ ?" } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
+ @usrv = map { "^" . _clean_regex_chars($$search{$_}{value}) } grep { ''.$$search{$_}{group} eq '0' } keys %$search;
+ }
my $addr = join ' AND ', map { "evergreen.lowercase(CAST($_ AS text)) ~ ?" } grep { ''.$$search{$_}{group} eq '1' } keys %$search;
my @addrv = map { "^" . _clean_regex_chars($$search{$_}{value}) } grep { ''.$$search{$_}{group} eq '1' } keys %$search;
@@ -712,7 +740,11 @@ sub patron_search {
my @namev;
if (0 && $nv) {
for my $n ( qw/first_given_name second_given_name family_name/ ) {
- push @ns, "evergreen.lowercase($n) ~ ?";
+ if ($diacritic_insensitive) {
+ push @ns, "evergreen.unaccent_and_squash($n) ~ ?";
+ } else {
+ push @ns, "evergreen.lowercase($n) ~ ?";
+ }
push @namev, "^$nv";
}
$name = '(' . join(' OR ', @ns) . ')';
commit 85a470ef7144265ee0a2618fafffeac8bb36b65d
Author: Dan Pearl <dpearl at cwmars.org>
Date: Thu Oct 1 09:49:17 2015 -0400
LP#1501781 - Make patron name search diacritic/space insensitive.
Diacritical marks may exist in the patron record, as they should, but
this makes patron name search difficult for librarians who may be
unfamiliar with all the special characters used and also how to elicit
them from a keyboard. To ease this, accented characters will be
converted into their 'plain' analogs for comparison purposes.
So, for example, if the patron's last name is Chávez, typing "Chavez"
in the Last Name box in Patron Search will match it.
Spaces in a name (like "De la Croix") will be squashed out so that
name would be matched by "Delacroix" or "De la Croix" or variants.
The librarian can enter the accented characters or not.
Signed-off-by: Dan Pearl <dpearl at cwmars.org>
Signed-off-by: Jason Stephenson <jason at sigio.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/000.functions.general.sql b/Open-ILS/src/sql/Pg/000.functions.general.sql
index ad6182e..9b95af8 100644
--- a/Open-ILS/src/sql/Pg/000.functions.general.sql
+++ b/Open-ILS/src/sql/Pg/000.functions.general.sql
@@ -86,4 +86,12 @@ END
$protect_reserved$
LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION evergreen.unaccent_and_squash ( IN arg text) RETURNS text
+ IMMUTABLE STRICT AS $$
+ BEGIN
+ RETURN evergreen.lowercase(unaccent(regexp_replace(arg, '\s','','g')));
+ END;
+$$ LANGUAGE PLPGSQL;
+
+
COMMIT;
diff --git a/Open-ILS/src/sql/Pg/005.schema.actors.sql b/Open-ILS/src/sql/Pg/005.schema.actors.sql
index da94af0..d7e3e6c 100644
--- a/Open-ILS/src/sql/Pg/005.schema.actors.sql
+++ b/Open-ILS/src/sql/Pg/005.schema.actors.sql
@@ -82,6 +82,9 @@ CREATE INDEX actor_usr_billing_address_idx ON actor.usr (billing_address);
CREATE INDEX actor_usr_first_given_name_idx ON actor.usr (evergreen.lowercase(first_given_name));
CREATE INDEX actor_usr_second_given_name_idx ON actor.usr (evergreen.lowercase(second_given_name));
CREATE INDEX actor_usr_family_name_idx ON actor.usr (evergreen.lowercase(family_name));
+CREATE INDEX actor_usr_first_given_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(first_given_name));
+CREATE INDEX actor_usr_second_given_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(second_given_name));
+CREATE INDEX actor_usr_family_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(family_name));
CREATE INDEX actor_usr_usrname_idx ON actor.usr (evergreen.lowercase(usrname));
CREATE INDEX actor_usr_email_idx ON actor.usr (evergreen.lowercase(email));
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 46825ea..c8960b1 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -16297,3 +16297,23 @@ INSERT INTO config.org_unit_setting_type
'coust', 'description'),
'integer');
+INSERT INTO config.org_unit_setting_type
+( name, grp, label, description, datatype )
+VALUES
+('circ.patron_search.diacritic_insensitive',
+ 'circ',
+ oils_i18n_gettext('circ.patron_search.diacritic_insensitive',
+ 'Patron search diacritic insensitive',
+ 'coust', 'label'),
+ oils_i18n_gettext('circ.patron_search.diacritic_insensitive',
+ 'Match patron last, first, and middle names irrespective of usage of diacritical marks or spaces. (e.g., Ines will match Inés; de la Cruz will match Delacruz)',
+ 'coust', 'description'),
+ 'bool');
+
+INSERT INTO actor.org_unit_setting (
+ org_unit, name, value
+) VALUES (
+ (SELECT id FROM actor.org_unit WHERE parent_ou IS NULL),
+ 'circ.patron_search.diacritic_insensitive',
+ 'true'
+);
diff --git a/Open-ILS/src/sql/Pg/create_database_extensions.sql b/Open-ILS/src/sql/Pg/create_database_extensions.sql
index ed0a964..013032c 100644
--- a/Open-ILS/src/sql/Pg/create_database_extensions.sql
+++ b/Open-ILS/src/sql/Pg/create_database_extensions.sql
@@ -20,3 +20,4 @@ CREATE EXTENSION xml2;
CREATE EXTENSION hstore;
CREATE EXTENSION intarray;
CREATE EXTENSION pgcrypto;
+CREATE EXTENSION unaccent;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql
new file mode 100644
index 0000000..3ec260b
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.patron_unaccent.sql
@@ -0,0 +1,43 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version);
+
+CREATE EXTENSION IF NOT EXISTS unaccent SCHEMA public;
+
+CREATE OR REPLACE FUNCTION evergreen.unaccent_and_squash ( IN arg text) RETURNS text
+ IMMUTABLE STRICT AS $$
+ BEGIN
+ RETURN evergreen.lowercase(unaccent(regexp_replace(arg, '\s','','g')));
+ END;
+$$ LANGUAGE PLPGSQL;
+
+-- The unaccented indices for patron name fields
+CREATE INDEX actor_usr_first_given_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(first_given_name));
+CREATE INDEX actor_usr_second_given_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(second_given_name));
+CREATE INDEX actor_usr_family_name_unaccent_idx ON actor.usr (evergreen.unaccent_and_squash(family_name));
+
+-- DB setting to control behavior; true by default
+INSERT INTO config.org_unit_setting_type
+( name, grp, label, description, datatype )
+VALUES
+('circ.patron_search.diacritic_insensitive',
+ 'circ',
+ oils_i18n_gettext('circ.patron_search.diacritic_insensitive',
+ 'Patron search diacritic insensitive',
+ 'coust', 'label'),
+ oils_i18n_gettext('circ.patron_search.diacritic_insensitive',
+ 'Match patron last, first, and middle names irrespective of usage of diacritical marks or spaces. (e.g., Ines will match Inés; de la Cruz will match Delacruz)',
+ 'coust', 'description'),
+ 'bool');
+
+INSERT INTO actor.org_unit_setting (
+ org_unit, name, value
+) VALUES (
+ (SELECT id FROM actor.org_unit WHERE parent_ou IS NULL),
+ 'circ.patron_search.diacritic_insensitive',
+ 'true'
+);
+
+
+COMMIT;
+
diff --git a/docs/RELEASE_NOTES_NEXT/Client/accent_insensitive_patron_search b/docs/RELEASE_NOTES_NEXT/Client/accent_insensitive_patron_search
new file mode 100644
index 0000000..fed2892
--- /dev/null
+++ b/docs/RELEASE_NOTES_NEXT/Client/accent_insensitive_patron_search
@@ -0,0 +1,20 @@
+Accent Insensitive Patron Search
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+When performing a patron search, in addition to the (existing)
+case-insensitivity, these additional characteristics will govern the
+search:
+
+Accent (diacritic) insensitivity::
+Diacritics will be transformed into a plain character equivalent for
+comparison purposes. So if the patron name is Eugène Delacroix, for
+example, you could enter euge for the First Name, and it would
+match. Ligatures such as Πare expanded into the constituent
+characters "OE".
+
+Space insensitivity::
+Spaces will be squashed out for comparison purposes. If the patron is,
+again, Eugène Delacroix, you could enter "de la croix" in the Last
+Name field and it would match.
+
+This behavior affects the Last Name, First Name, and Middle Name fields
+of the search.
-----------------------------------------------------------------------
Summary of changes:
.../OpenILS/Application/Storage/Publisher/actor.pm | 40 ++++++++++++-
Open-ILS/src/sql/Pg/000.functions.general.sql | 8 +++
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/005.schema.actors.sql | 3 +
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 20 +++++++
Open-ILS/src/sql/Pg/create_database_extensions.sql | 1 +
.../src/sql/Pg/t/lp1501781-unaccent_and_squash.pg | 61 ++++++++++++++++++++
.../sql/Pg/upgrade/0986.schema.patron_unaccent.sql | 43 ++++++++++++++
.../Client/accent_insensitive_patron_search | 20 +++++++
9 files changed, 193 insertions(+), 5 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/t/lp1501781-unaccent_and_squash.pg
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0986.schema.patron_unaccent.sql
create mode 100644 docs/RELEASE_NOTES_NEXT/Client/accent_insensitive_patron_search
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list