[open-ils-commits] [GIT] Evergreen ILS branch master updated. 2644ddda0779f36a6a7955956d117741bd902ee9
Evergreen Git
git at git.evergreen-ils.org
Wed Feb 8 15:15:18 EST 2017
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 2644ddda0779f36a6a7955956d117741bd902ee9 (commit)
via d107f15a29775fa88526f3feb2fccb23083f7074 (commit)
via dcc74b384f08fa95638698296e07e4b0468c15a2 (commit)
via fd7f90415ab5302c32b0a70550d7e5d16b03e73b (commit)
from a6f1a4f27402a1b255b6876df76b73f33238af4e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 2644ddda0779f36a6a7955956d117741bd902ee9
Author: Kathy Lussier <klussier at masslnc.org>
Date: Wed Feb 8 15:11:19 2017 -0500
LP#1308090: Stamping upgrade script for trim trailing punctuation normalizer
Signed-off-by: Kathy Lussier <klussier at masslnc.org>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index c0b4613..6c61454 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -91,7 +91,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1005', :eg_version); -- csharp/berick
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1006', :eg_version); -- DPearl/kmlussier
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql b/Open-ILS/src/sql/Pg/upgrade/1006.function.trim_trailing_punctuation.sql
similarity index 95%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql
rename to Open-ILS/src/sql/Pg/upgrade/1006.function.trim_trailing_punctuation.sql
index b13f66f..5a4004a 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/1006.function.trim_trailing_punctuation.sql
@@ -1,3 +1,5 @@
+SELECT evergreen.upgrade_deps_block_check('1006', :eg_version);
+
BEGIN;
-- This function is used to help clean up facet labels. Due to quirks in
commit d107f15a29775fa88526f3feb2fccb23083f7074
Author: Kathy Lussier <klussier at masslnc.org>
Date: Wed Feb 8 14:19:08 2017 -0500
LP#1308090: Updating release notes to reflect both parts of this new feature
The code introduced two distinct features related to authors and RDA records,
but only one was highlighted in the release notes. Notes are updated to reflect
both changes.
Signed-off-by: Kathy Lussier <klussier at masslnc.org>
diff --git a/docs/RELEASE_NOTES_NEXT/OPAC/relator_list b/docs/RELEASE_NOTES_NEXT/OPAC/relator_list
deleted file mode 100644
index bfe02aa..0000000
--- a/docs/RELEASE_NOTES_NEXT/OPAC/relator_list
+++ /dev/null
@@ -1,5 +0,0 @@
-Author Roles
-^^^^^^^^^^^^
-All author/contrbutor roles will now be displayed in the record detail. Previously, some
-of the roles were omitted.
-
diff --git a/docs/RELEASE_NOTES_NEXT/OPAC/relator_list.adoc b/docs/RELEASE_NOTES_NEXT/OPAC/relator_list.adoc
new file mode 100644
index 0000000..d364f84
--- /dev/null
+++ b/docs/RELEASE_NOTES_NEXT/OPAC/relator_list.adoc
@@ -0,0 +1,8 @@
+RDA Improvements
+^^^^^^^^^^^^^^^^
+ * Author fields are now normalized to strip ending periods so that authors from
+RDA and non-RDA records are collapsed in browse and facet headings.
+ * All author/contrbutor roles will now be displayed in the record detail.
+Previously, some of the roles were omitted or were duplicated.
+
+This new feature will require a record reingest.
commit dcc74b384f08fa95638698296e07e4b0468c15a2
Author: Kathy Lussier <klussier at masslnc.org>
Date: Mon Oct 3 12:25:58 2016 -0400
LP#1308090: pgTAP fixes
Signed-off-by: Kathy Lussier <klussier at masslnc.org>
Signed-off-by: Dan Pearl <dpearl at cwmars.org>
LP#1308090 - Additional fix to pgtap test.
Signed-off-by: Dan Pearl <dpearl at cwmars.org>
Signed-off-by: Kathy Lussier <klussier at masslnc.org>
diff --git a/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg b/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
index acf6846..07e69f6 100644
--- a/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
+++ b/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
@@ -9,13 +9,13 @@ SELECT is( metabib.trim_trailing_punctuation(''), '', 'Empty string');
SELECT is( metabib.trim_trailing_punctuation('X,'), 'X', 'Eliminate comma A');
SELECT is( metabib.trim_trailing_punctuation('Smith, John,'), 'Smith, John', 'Eliminate comma B');
-SELECT is( metabib.trim_trailing_punctuation('X.'), 'X.', 'Initial w/o preceding space (period)');
+SELECT is( metabib.trim_trailing_punctuation('X.'), 'X', 'Initial w/o preceding space (period)');
SELECT is( metabib.trim_trailing_punctuation('X@'), 'X@', 'Initial w/o preceding space (other)');
SELECT is( metabib.trim_trailing_punctuation('Smith, John'), 'Smith, John', 'Name no trailing punct A');
SELECT is( metabib.trim_trailing_punctuation('Saki'), 'Saki', 'Name no trailing punct B');
SELECT is( metabib.trim_trailing_punctuation('Smith, John.'), 'Smith, John', 'Chop trailing period');
-SELECT is( metabib.trim_trailing_punctuation('Public, John Q.'), 'Pulbic, John Q.', 'Retain trailing period');
-SELECT is( metabib.trim_trailing_punctuation('Public, John Q,'), 'Pulbic, John Q', 'Eliminate comma C');
+SELECT is( metabib.trim_trailing_punctuation('Public, John Q.'), 'Public, John Q.', 'Retain trailing period');
+SELECT is( metabib.trim_trailing_punctuation('Public, John Q,'), 'Public, John Q', 'Eliminate comma C');
SELECT is( metabib.trim_trailing_punctuation('(FTC).'), '(FTC)', 'Trailing period');
SELECT * FROM finish();
commit fd7f90415ab5302c32b0a70550d7e5d16b03e73b
Author: Dan Pearl <dpearl at cwmars.org>
Date: Thu Jun 2 15:02:25 2016 -0400
LP#1308090 Relator fields and facets need normalization.
This incorporates two changes: facets were generated with trailing
punctuation, which resulted in more than one entry for the same item,
differing only in punctuation. In addition, relator codes were suppressed
in the record detail unnecessarily.
Signed-off-by: Dan Pearl <dpearl at cwmars.org>
Signed-off-by: Kate Butler <katebutler at rodgerslibrary.org>
Signed-off-by: Kathy Lussier <klussier at masslnc.org>
diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index d732f4a..71bab56 100644
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -2458,5 +2458,35 @@ BEGIN
END;
$p$ LANGUAGE PLPGSQL;
+-- This function is used to help clean up facet labels. Due to quirks in
+-- MARC parsing, some facet labels may be generated with periods or commas
+-- at the end. This will strip a trailing commas off all the time, and
+-- periods when they don't look like they are part of initials.
+-- Smith, John => no change
+-- Smith, John, => Smith, John
+-- Smith, John. => Smith, John
+-- Public, John Q. => no change
+CREATE OR REPLACE FUNCTION metabib.trim_trailing_punctuation ( TEXT ) RETURNS TEXT AS $$
+DECLARE
+ result TEXT;
+ last_char TEXT;
+BEGIN
+ result := $1;
+ last_char = substring(result from '.$');
+
+ IF last_char = ',' THEN
+ result := substring(result from '^(.*),$');
+
+ ELSIF last_char = '.' THEN
+ IF substring(result from ' \w\.$') IS NULL THEN
+ result := substring(result from '^(.*)\.$');
+ END IF;
+ END IF;
+
+ RETURN result;
+
+END;
+$$ language 'plpgsql';
+
COMMIT;
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index e029263..5e76959 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -10046,6 +10046,13 @@ INSERT INTO config.index_normalizer (name, description, func, param_count) VALUE
0
);
+INSERT INTO config.index_normalizer (name, description, func, param_count) VALUES (
+ 'Trim Trailing Punctuation',
+ 'Eliminate extraneous trailing commas and periods in text',
+ 'metabib.trim_trailing_punctuation',
+ 0
+);
+
-- make use of the index normalizers
INSERT INTO config.metabib_field_index_norm_map (field,norm)
@@ -10103,6 +10110,16 @@ INSERT INTO config.metabib_field_index_norm_map (field,norm,pos)
WHERE i.func = 'remove_paren_substring'
AND m.id IN (28);
+INSERT INTO config.metabib_field_index_norm_map (field,norm,pos)
+ SELECT m.id,
+ i.id,
+ -1
+ FROM config.metabib_field m,
+ config.index_normalizer i
+ WHERE i.func = 'metabib.trim_trailing_punctuation'
+ AND m.id IN (7,8,9,10);
+
+
INSERT INTO config.record_attr_index_norm_map (attr,norm,pos)
SELECT m.name, i.id, 0
FROM config.record_attr_definition m,
diff --git a/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg b/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
new file mode 100644
index 0000000..acf6846
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
@@ -0,0 +1,23 @@
+BEGIN;
+
+SELECT plan(12);
+
+SELECT can('metabib', ARRAY['trim_trailing_punctuation'], 'metabib.trim_trailing_punctuation function exists');
+
+SELECT is( metabib.trim_trailing_punctuation(''), '', 'Empty string');
+
+SELECT is( metabib.trim_trailing_punctuation('X,'), 'X', 'Eliminate comma A');
+SELECT is( metabib.trim_trailing_punctuation('Smith, John,'), 'Smith, John', 'Eliminate comma B');
+
+SELECT is( metabib.trim_trailing_punctuation('X.'), 'X.', 'Initial w/o preceding space (period)');
+SELECT is( metabib.trim_trailing_punctuation('X@'), 'X@', 'Initial w/o preceding space (other)');
+SELECT is( metabib.trim_trailing_punctuation('Smith, John'), 'Smith, John', 'Name no trailing punct A');
+SELECT is( metabib.trim_trailing_punctuation('Saki'), 'Saki', 'Name no trailing punct B');
+SELECT is( metabib.trim_trailing_punctuation('Smith, John.'), 'Smith, John', 'Chop trailing period');
+SELECT is( metabib.trim_trailing_punctuation('Public, John Q.'), 'Pulbic, John Q.', 'Retain trailing period');
+SELECT is( metabib.trim_trailing_punctuation('Public, John Q,'), 'Pulbic, John Q', 'Eliminate comma C');
+SELECT is( metabib.trim_trailing_punctuation('(FTC).'), '(FTC)', 'Trailing period');
+
+SELECT * FROM finish();
+
+ROLLBACK;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql
new file mode 100644
index 0000000..b13f66f
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.function.trim_trailing_punctuation.sql
@@ -0,0 +1,50 @@
+BEGIN;
+
+-- This function is used to help clean up facet labels. Due to quirks in
+-- MARC parsing, some facet labels may be generated with periods or commas
+-- at the end. This will strip a trailing commas off all the time, and
+-- periods when they don't look like they are part of initials.
+-- Smith, John => no change
+-- Smith, John, => Smith, John
+-- Smith, John. => Smith, John
+-- Public, John Q. => no change
+CREATE OR REPLACE FUNCTION metabib.trim_trailing_punctuation ( TEXT ) RETURNS TEXT AS $$
+DECLARE
+ result TEXT;
+ last_char TEXT;
+BEGIN
+ result := $1;
+ last_char = substring(result from '.$');
+
+ IF last_char = ',' THEN
+ result := substring(result from '^(.*),$');
+
+ ELSIF last_char = '.' THEN
+ IF substring(result from ' \w\.$') IS NULL THEN
+ result := substring(result from '^(.*)\.$');
+ END IF;
+ END IF;
+
+ RETURN result;
+
+END;
+$$ language 'plpgsql';
+
+INSERT INTO config.index_normalizer (name, description, func, param_count) VALUES (
+ 'Trim Trailing Punctuation',
+ 'Eliminate extraneous trailing commas and periods in text',
+ 'metabib.trim_trailing_punctuation',
+ 0
+);
+
+INSERT INTO config.metabib_field_index_norm_map (field,norm,pos)
+ SELECT m.id,
+ i.id,
+ -1
+ FROM config.metabib_field m,
+ config.index_normalizer i
+ WHERE i.func = 'metabib.trim_trailing_punctuation'
+ AND m.id IN (7,8,9,10);
+
+COMMIT;
+
diff --git a/Open-ILS/src/templates/opac/parts/record/authors.tt2 b/Open-ILS/src/templates/opac/parts/record/authors.tt2
index 735b3ef..2208cf0 100644
--- a/Open-ILS/src/templates/opac/parts/record/authors.tt2
+++ b/Open-ILS/src/templates/opac/parts/record/authors.tt2
@@ -25,6 +25,7 @@ authors = [
BLOCK normalize_qterm;
subfield.textContent.replace('[#"^$\+\-,\.:;&|\[\]()]', ' ');
+ subfield.textContent.replace('\s{2,}', ' ');
END;
BLOCK normalize_authors;
@@ -42,7 +43,7 @@ BLOCK build_author_links;
link_term = ''; # Linked term (e.g. Personal name + Fuller form of name)
supp_term = ''; # Supplementary terms
qterm = ''; # Search query
- tlabel = '';
+ tlabels = [];
birthdate = '';
deathdate = '';
graphics = [];
@@ -53,7 +54,11 @@ BLOCK build_author_links;
code = subfield.getAttribute('code');
IF code == '4';
relcode = subfield.textContent.substr(0,3);
- tlabel = relators.$relcode || label;
+ tlabels.push( relators.$relcode || label );
+ END;
+ IF code == 'e';
+ tlabels.push( subfield.textContent() );
+ indexed_term = 1;
END;
IF code == '6';
target_field = tag;
@@ -92,6 +97,8 @@ BLOCK build_author_links;
END;
END;
url = mkurl(ctx.opac_root _ '/results', {query => qterm.replace('^\s*(.*?)\s*$', '$1'), qtype => 'author'}, stop_parms.merge(expert_search_parms, general_search_parms, browse_search_parms, facet_search_parms));
+ tlabel = tlabels.join(', ');
+ tlabels = [];
author_type = (tlabel || label) | html;
# schema.org changes
diff --git a/docs/RELEASE_NOTES_NEXT/OPAC/relator_list b/docs/RELEASE_NOTES_NEXT/OPAC/relator_list
new file mode 100644
index 0000000..bfe02aa
--- /dev/null
+++ b/docs/RELEASE_NOTES_NEXT/OPAC/relator_list
@@ -0,0 +1,5 @@
+Author Roles
+^^^^^^^^^^^^
+All author/contrbutor roles will now be displayed in the record detail. Previously, some
+of the roles were omitted.
+
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/030.schema.metabib.sql | 30 +++++++++++
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 17 ++++++
Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg | 23 +++++++++
.../1006.function.trim_trailing_punctuation.sql | 52 ++++++++++++++++++++
.../src/templates/opac/parts/record/authors.tt2 | 11 +++-
docs/RELEASE_NOTES_NEXT/OPAC/relator_list.adoc | 8 +++
7 files changed, 140 insertions(+), 3 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/t/lp1308090-facet_punct.pg
create mode 100644 Open-ILS/src/sql/Pg/upgrade/1006.function.trim_trailing_punctuation.sql
create mode 100644 docs/RELEASE_NOTES_NEXT/OPAC/relator_list.adoc
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list