[open-ils-commits] [GIT] Evergreen ILS branch master updated. 433ffa2a57f0452f795484bfed96d74499407dc0
Evergreen Git
git at git.evergreen-ils.org
Fri Feb 15 15:44:06 EST 2013
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 433ffa2a57f0452f795484bfed96d74499407dc0 (commit)
via 7b8fc7b67a3c7cf3386df17950f686cef6ec5998 (commit)
via 0817eccf2a805034e2b8f0c358456c0ef93f4aa8 (commit)
via 7da2227f1ee3c98b98b518fa2ca4f857b9173a88 (commit)
via b3a485dd49266d7e6d26bf9c25d7fef9cdced164 (commit)
via e1149c69cf994f9f04441056b7e0b0538ac3d0ce (commit)
via 3249c78b060fb6bcf00964b825c3a77c332f73a4 (commit)
via 6a90104750160b087a0252d6775ce5eb61d533d5 (commit)
via bdbec2aadf744331c25c27983402acbdfbe8396a (commit)
via 0662b43a449f33fbb659b94a222da099d342a19c (commit)
via b4fb02f964b88d4848a2fb0e2d242ed8b3cb5fcf (commit)
via c05bd342e188daa04972d31c00909b81d056e78c (commit)
via 0db54b56c88e540a44f1af2dbec126dd3c6409ba (commit)
via a4f5bc9dce8fb4065879db02559ef044d31ea888 (commit)
via 5aa5e26f4053e7193f7cf8f9b6845f027ce34917 (commit)
via 60a420efd8e043439684af693ca331b023106924 (commit)
via f5a4c11716fbcf5a248dfa69533fbc660e646e0c (commit)
via 52d16172cf8eed5b11086b01361cfe9def4dc8be (commit)
via ab9fb958e387a20cfe9fafb6035fb72bc5f1fb3f (commit)
via 613a6da032ab0d177421c36fe07d1d1dd9c6922c (commit)
via 6d8872cf120caf67ad6f65995b2c5155fa5ab652 (commit)
via c7c3d1bcfd7e394f5698ea0615ad126d71741693 (commit)
via 7e2dd736ffe0dbc969ce4e365efe8834889a103a (commit)
via 9626889ea9c693b2576593591417dcbf11306f93 (commit)
via 1199e3835f8308ca5a9d9b5329a60594e4710ef5 (commit)
via 9c2df12c20f73619f427fca20fa4b79e167df35c (commit)
via 56d46e45f58616ab831247f7c6858de55e35962e (commit)
via 8a709bf9ca16355d74f1791483e23a13aea03ed6 (commit)
via efa0f86ee926d8f3e1068779b3e01eb0943c9a57 (commit)
via 1cdbcb8eccbeec914aeeb05876cc44d164c2052c (commit)
via ba2ad7bda934b2184eba42dcd1eb1860bbcd6599 (commit)
via be608c694172d6536b8d48efea5bae4c338fdca6 (commit)
via 264a90828359118a3b736e8de4a14a450997b4eb (commit)
via 1d5ed2a3a1d6eba163d6a92866b2cdeef8ad5165 (commit)
via cdb64b8159ec7edf920bf86dfef2fad96fe12fdf (commit)
via bdcfdfb259c33d11086e8732e1e689a60d2828cc (commit)
via 205ea5125eb0c22932ea5774d299d7cac2ba3301 (commit)
from bea6cffd710e3f1e05803d8152f41dbe7635f328 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 433ffa2a57f0452f795484bfed96d74499407dc0
Author: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
Date: Fri Feb 15 15:36:01 2013 -0500
qp_fix: number upgrade scripts, disable QP unit tests needing more attention
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/t/21-QueryParser.t b/Open-ILS/src/perlmods/t/21-QueryParser.t
index 2427999..55ffd6f 100644
--- a/Open-ILS/src/perlmods/t/21-QueryParser.t
+++ b/Open-ILS/src/perlmods/t/21-QueryParser.t
@@ -53,11 +53,20 @@ is($QParser->core_limit(), 25000, 'Core limit stays set');
is($QParser->superpage(1), 1, 'Superpage setting works');
is($QParser->superpage(), 1, 'Superpage stays set');
-is($QParser->superpage(0), 0, 'Superpage can be unset');
+
+# see QueryParser.pm, this won't work:
+# is($QParser->superpage(0), 0, 'Superpage can be unset');
is($QParser->superpage_size(1000), 1000, 'Superpage size setting works');
is($QParser->superpage_size(), 1000, 'Superpage size stays set');
+# It's unfortunate not to be able to use the following tests immediately, but
+# they reflect assumptions that need to be updated in light of new qp_fix code.
+# Also,, canonicalization may not preserve insignificant whitespace nor the
+# exact, original number of non-semantic parentheses.
+
+=cut
+
init_qp();
my %queries = (
@@ -149,6 +158,7 @@ while (($query, $different) = each (%differences)) {
isnt($canonical1, $canonical2, "Queries {$query} and {$different} are not equivalent");
}
+=cut
done_testing;
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 11dd989..af1a771 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -87,7 +87,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0755', :eg_version); -- berick/bshum
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0757', :eg_version); -- tsbere/senator
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql b/Open-ILS/src/sql/Pg/upgrade/0756.drop.query_parser_fts.sql
similarity index 69%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql
rename to Open-ILS/src/sql/Pg/upgrade/0756.drop.query_parser_fts.sql
index ee76bd1..2314d91 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0756.drop.query_parser_fts.sql
@@ -1,3 +1,9 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('0756', :eg_version);
+
DROP FUNCTION IF EXISTS search.query_parser_fts(INT,INT,TEXT,INT[],INT[],INT,INT,INT,BOOL,BOOL,INT);
DROP TYPE IF EXISTS search.search_result;
DROP TYPE IF EXISTS search.search_args;
+
+COMMIT;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql b/Open-ILS/src/sql/Pg/upgrade/0757.schema.ts_configs.sql
similarity index 99%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql
rename to Open-ILS/src/sql/Pg/upgrade/0757.schema.ts_configs.sql
index f36d447..c98f038 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/0757.schema.ts_configs.sql
@@ -1,5 +1,7 @@
BEGIN;
+SELECT evergreen.upgrade_deps_block_check('0757', :eg_version);
+
SET search_path = public, pg_catalog;
DO $$
commit 7b8fc7b67a3c7cf3386df17950f686cef6ec5998
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Dec 18 15:53:28 2012 -0500
QueryParser: use combined metabib_field column
Because I somehow forgot about it. Oops.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index b5fc794..8914bd1 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -998,41 +998,42 @@ sub flatten {
. ${spc} x 6 . "FROM $table AS fe";
$from .= "\n" . ${spc} x 7 . "JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
- if ($node->dummy_count < @{$node->only_atoms} ) {
- $with .= ",\n " if $with;
- $with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq,". $node->tsquery_rank ." AS tsq_rank )";
- $from .= "\n" . ${spc} x 6 . "JOIN $ctable AS com ON (com.record = fe.source)";
- if (@{$node->fields} > 0) {
- $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (com.index_vector @@ ${talias}_xq.tsq_rank AND fe.index_vector @@ ${talias}_xq.tsq)";
- } else {
- $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (com.index_vector @@ ${talias}_xq.tsq)";
- }
- } else {
- $from .= "\n" . ${spc} x 6 . ", (SELECT NULL::tsquery AS tsq, NULL:tsquery AS tsq_rank ) AS ${talias}_xq";
- }
-
my @bump_fields;
+ my @field_ids;
if (@{$node->fields} > 0) {
@bump_fields = @{$node->fields};
- my @field_ids = grep defined, (
+ @field_ids = grep defined, (
map {
$self->QueryParser->search_field_ids_by_class(
$node->classname, $_
)->[0]
} @bump_fields
);
+ } else {
+ @bump_fields = @{$self->QueryParser->search_fields->{$node->classname}};
+ }
+
+ if ($node->dummy_count < @{$node->only_atoms} ) {
+ $with .= ",\n " if $with;
+ $with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq,". $node->tsquery_rank ." AS tsq_rank )";
+ $from .= "\n" . ${spc} x 6 . "JOIN $ctable AS com ON (com.record = fe.source";
if (@field_ids) {
- $from .= "\n" . ${spc} x 6 . "WHERE fe_weight.id IN (" .
- join(',', @field_ids) . ")";
+ $from .= " AND com.metabib_field IN (" . join(',', at field_ids) . "))";
+ } else {
+ $from .= " AND com.metabib_field IS NULL)";
}
-
+ $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (com.index_vector @@ ${talias}_xq.tsq)";
} else {
- @bump_fields = @{$self->QueryParser->search_fields->{$node->classname}};
+ $from .= "\n" . ${spc} x 6 . ", (SELECT NULL::tsquery AS tsq, NULL:tsquery AS tsq_rank ) AS ${talias}_xq";
}
- $from .= "\n" . ${spc} x 4 . ") AS $talias ON (m.source = ${talias}.source)";
+ if (@field_ids) {
+ $from .= "\n" . ${spc} x 6 . "WHERE fe_weight.id IN (" .
+ join(',', @field_ids) . ")";
+ }
+ $from .= "\n" . ${spc} x 4 . ") AS $talias ON (m.source = ${talias}.source)";
my %used_bumps;
my @bumps;
commit 0817eccf2a805034e2b8f0c358456c0ef93f4aa8
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Dec 18 15:12:10 2012 -0500
Add basic release notes and notes for docwriters
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/docs/QueryParser_Changes.txt b/docs/QueryParser_Changes.txt
new file mode 100644
index 0000000..02fef40
--- /dev/null
+++ b/docs/QueryParser_Changes.txt
@@ -0,0 +1,47 @@
+QueryParser Changes
+
+Quick notes for doc writers.
+
+New columns:
+
+config.metabib_class: Note: This gets a new config interface to expose this and other information. It intentionally has no buttons for adding or removing entries.
+ a_weight
+ b_weight
+ c_weight
+ d_weight
+
+These are the FTS weights used for ranking for the four FTS weight classes. By default "A" is the exact match indexing and "C" is the stemmed version. They default to the PostgreSQL defaults that are used when otherwise unspecified.
+
+
+New tables:
+
+config.ts_config_list: Note: No editing interface exists for this, intentionally. It should be added to when DB-level FTS configs are added.
+ id - Actual DB level text search config name
+ name - Human readable description
+
+This lists the valid FTS configs for use with the following two tables, with more human friendly names.
+
+config.metabib_class_ts_map: Editable from the Server Admin menu
+ id - Primary key for editor benefit
+ field_class - Reference to config.metabib_class
+ ts_config - Which Text Search config to use
+ active - Is this config active. If false will not be used for searching or indexing.
+ index_weight - The FTS index weight to use for this FTS config. Should be A, B, C, or D, defaults to C.
+ index_lang - If set what language the record should be set to in order for this FTS config to be used for indexing
+ search_lang - If set what preferred language search should be using in order for this FTS config to be used for searching
+ always - If true use this config even when searching a specific field (author|personal, for example) even if that field has config as well
+
+This maps broad search classes and text search configs. Multiple can exist for a given search class. Setting index_lang or search_lang to 'NONE' will effectively disable the config for that purpose as they check against a three character field like 'eng' or 'fre'.
+
+config.metabib_field_ts_map: Editable from the Server Admin menu
+ id - Primary key for editor benefit
+ metabib_field - Reference to config.metabib_field
+ ts_config - Which Text Serach config to use
+ active - Is this config active. If false will not be used for searching or indexing.
+ index_weight - The FTS index weight to use for this FTS config. Should be A, B, C, or D, defaults to C.
+ index_lang - If set what language the record should be set to in order for this FTS config to be used for indexing
+ search_lang - If set what preferred language search should be using in order for this FTS config to be used for searching
+
+This maps individual indexes and text search configs. Multiple can exist for a given index. Setting index_lang or search_lang to 'NONE' will effectively disable the config for that purpose as they check against a three character field like 'eng' or 'fre'. Note that anything from the broader configs will be used if none exist for the specified field and the "always" ones from the broader configs will be used even if field specific ones do exist.
+
+New non-configuration tables exist for combined search indexes, but they are, IMO, more implementation details than things to be documented for end users.
diff --git a/docs/RELEASE_NOTES_NEXT/queryparser_changes.txt b/docs/RELEASE_NOTES_NEXT/queryparser_changes.txt
new file mode 100644
index 0000000..d256096
--- /dev/null
+++ b/docs/RELEASE_NOTES_NEXT/queryparser_changes.txt
@@ -0,0 +1,25 @@
+Search Changes
+==============
+A number of changes have been made to search to allow more control and improve
+performance. These changes and their associated configurations are global to
+the entire system and can not be configured on a per-library basis.
+
+Amongst other things the new search code fixes:
+
+* Inability to use statuses and locations as part of a larger query
+* Invalid queries being generated from advanced searches
+* Some timeouts from backend code taking too long to preform a search
+* Some filters being one-use only
+* Negations not working properly where multiple indexes are involved
+
+Improvements include:
+
+* Exact matches on input should be more likely to float to the top of results
+* Non-English stemming can be used, alongside or instead of English stemming
+* Entered search terms can be found across multiple indexes
+
+Default configuration is geared towards English but is easily changed. In a
+production environment changes will likely require re-indexing, however.
+
+The upgrade script could be pre-tweaked to install desired configuration before
+it builds and/or re-builds many of the indexes.
commit 7da2227f1ee3c98b98b518fa2ca4f857b9173a88
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Dec 18 14:35:28 2012 -0500
Upgrade script for ts config use
And the rest of that work
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql
new file mode 100644
index 0000000..f36d447
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.ts_configs.sql
@@ -0,0 +1,432 @@
+BEGIN;
+
+SET search_path = public, pg_catalog;
+
+DO $$
+DECLARE
+lang TEXT;
+BEGIN
+FOR lang IN SELECT substring(pptsd.dictname from '(.*)_stem$') AS lang FROM pg_catalog.pg_ts_dict pptsd JOIN pg_catalog.pg_namespace ppn ON ppn.oid = pptsd.dictnamespace
+WHERE ppn.nspname = 'pg_catalog' AND pptsd.dictname LIKE '%_stem' LOOP
+RAISE NOTICE 'FOUND LANGUAGE %', lang;
+
+EXECUTE 'DROP TEXT SEARCH DICTIONARY IF EXISTS ' || lang || '_nostop CASCADE;
+CREATE TEXT SEARCH DICTIONARY ' || lang || '_nostop (TEMPLATE=pg_catalog.snowball, language=''' || lang || ''');
+COMMENT ON TEXT SEARCH DICTIONARY ' || lang || '_nostop IS ''' ||lang || ' snowball stemmer with no stopwords for ASCII words only.'';
+CREATE TEXT SEARCH CONFIGURATION ' || lang || '_nostop ( COPY = pg_catalog.' || lang || ' );
+ALTER TEXT SEARCH CONFIGURATION ' || lang || '_nostop ALTER MAPPING FOR word, hword, hword_part WITH pg_catalog.simple;
+ALTER TEXT SEARCH CONFIGURATION ' || lang || '_nostop ALTER MAPPING FOR asciiword, asciihword, hword_asciipart WITH ' || lang || '_nostop;';
+
+END LOOP;
+END;
+$$;
+CREATE TEXT SEARCH CONFIGURATION keyword ( COPY = english_nostop );
+CREATE TEXT SEARCH CONFIGURATION "default" ( COPY = english_nostop );
+
+SET search_path = evergreen, public, pg_catalog;
+
+ALTER TABLE config.metabib_class
+ ADD COLUMN a_weight NUMERIC DEFAULT 1.0 NOT NULL,
+ ADD COLUMN b_weight NUMERIC DEFAULT 0.4 NOT NULL,
+ ADD COLUMN c_weight NUMERIC DEFAULT 0.2 NOT NULL,
+ ADD COLUMN d_weight NUMERIC DEFAULT 0.1 NOT NULL;
+
+CREATE TABLE config.ts_config_list (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL
+);
+COMMENT ON TABLE config.ts_config_list IS $$
+Full Text Configs
+
+A list of full text configs with names and descriptions.
+$$;
+
+CREATE TABLE config.metabib_class_ts_map (
+ id SERIAL PRIMARY KEY,
+ field_class TEXT NOT NULL REFERENCES config.metabib_class (name),
+ ts_config TEXT NOT NULL REFERENCES config.ts_config_list (id),
+ active BOOL NOT NULL DEFAULT TRUE,
+ index_weight CHAR(1) NOT NULL DEFAULT 'C' CHECK (index_weight IN ('A','B','C','D')),
+ index_lang TEXT NULL,
+ search_lang TEXT NULL,
+ always BOOL NOT NULL DEFAULT true
+);
+COMMENT ON TABLE config.metabib_class_ts_map IS $$
+Text Search Configs for metabib class indexing
+
+This table contains text search config definitions for
+storing index_vector values.
+$$;
+
+CREATE TABLE config.metabib_field_ts_map (
+ id SERIAL PRIMARY KEY,
+ metabib_field INT NOT NULL REFERENCES config.metabib_field (id),
+ ts_config TEXT NOT NULL REFERENCES config.ts_config_list (id),
+ active BOOL NOT NULL DEFAULT TRUE,
+ index_weight CHAR(1) NOT NULL DEFAULT 'C' CHECK (index_weight IN ('A','B','C','D')),
+ index_lang TEXT NULL,
+ search_lang TEXT NULL
+);
+COMMENT ON TABLE config.metabib_field_ts_map IS $$
+Text Search Configs for metabib field indexing
+
+This table contains text search config definitions for
+storing index_vector values.
+$$;
+
+CREATE TABLE metabib.combined_identifier_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_identifier_field_entry_fakepk_idx ON metabib.combined_identifier_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_identifier_field_entry_index_vector_idx ON metabib.combined_identifier_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_identifier_field_source_idx ON metabib.combined_identifier_field_entry (metabib_field);
+
+CREATE TABLE metabib.combined_title_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_title_field_entry_fakepk_idx ON metabib.combined_title_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_title_field_entry_index_vector_idx ON metabib.combined_title_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_title_field_source_idx ON metabib.combined_title_field_entry (metabib_field);
+
+CREATE TABLE metabib.combined_author_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_author_field_entry_fakepk_idx ON metabib.combined_author_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_author_field_entry_index_vector_idx ON metabib.combined_author_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_author_field_source_idx ON metabib.combined_author_field_entry (metabib_field);
+
+CREATE TABLE metabib.combined_subject_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_subject_field_entry_fakepk_idx ON metabib.combined_subject_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_subject_field_entry_index_vector_idx ON metabib.combined_subject_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_subject_field_source_idx ON metabib.combined_subject_field_entry (metabib_field);
+
+CREATE TABLE metabib.combined_keyword_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_keyword_field_entry_fakepk_idx ON metabib.combined_keyword_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_keyword_field_entry_index_vector_idx ON metabib.combined_keyword_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_keyword_field_source_idx ON metabib.combined_keyword_field_entry (metabib_field);
+
+CREATE TABLE metabib.combined_series_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_series_field_entry_fakepk_idx ON metabib.combined_series_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_series_field_entry_index_vector_idx ON metabib.combined_series_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_series_field_source_idx ON metabib.combined_series_field_entry (metabib_field);
+
+CREATE OR REPLACE FUNCTION metabib.update_combined_index_vectors(bib_id BIGINT) RETURNS VOID AS $func$
+BEGIN
+ DELETE FROM metabib.combined_keyword_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_keyword_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.keyword_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_keyword_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.keyword_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_title_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_title_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.title_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_title_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.title_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_author_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_author_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.author_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_author_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.author_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_subject_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_subject_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.subject_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_subject_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.subject_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_series_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_series_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.series_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_series_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.series_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_identifier_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_identifier_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.identifier_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_identifier_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.identifier_field_entry WHERE source = bib_id;
+
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$
+DECLARE
+ fclass RECORD;
+ ind_data metabib.field_entry_template%ROWTYPE;
+ mbe_row metabib.browse_entry%ROWTYPE;
+ mbe_id BIGINT;
+BEGIN
+ PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+ IF NOT FOUND THEN
+ IF NOT skip_search THEN
+ FOR fclass IN SELECT * FROM config.metabib_class LOOP
+ -- RAISE NOTICE 'Emptying out %', fclass.name;
+ EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+ END LOOP;
+ END IF;
+ IF NOT skip_facet THEN
+ DELETE FROM metabib.facet_entry WHERE source = bib_id;
+ END IF;
+ IF NOT skip_browse THEN
+ DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id;
+ END IF;
+ END IF;
+
+ FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+ IF ind_data.field < 0 THEN
+ ind_data.field = -1 * ind_data.field;
+ END IF;
+
+ IF ind_data.facet_field AND NOT skip_facet THEN
+ INSERT INTO metabib.facet_entry (field, source, value)
+ VALUES (ind_data.field, ind_data.source, ind_data.value);
+ END IF;
+
+ IF ind_data.browse_field AND NOT skip_browse THEN
+ -- A caveat about this SELECT: this should take care of replacing
+ -- old mbe rows when data changes, but not if normalization (by
+ -- which I mean specifically the output of
+ -- evergreen.oils_tsearch2()) changes. It may or may not be
+ -- expensive to add a comparison of index_vector to index_vector
+ -- to the WHERE clause below.
+ SELECT INTO mbe_row * FROM metabib.browse_entry WHERE value = ind_data.value;
+ IF FOUND THEN
+ mbe_id := mbe_row.id;
+ ELSE
+ INSERT INTO metabib.browse_entry (value) VALUES
+ (metabib.browse_normalize(ind_data.value, ind_data.field));
+ mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS);
+ END IF;
+
+ INSERT INTO metabib.browse_entry_def_map (entry, def, source)
+ VALUES (mbe_id, ind_data.field, ind_data.source);
+ END IF;
+
+ IF ind_data.search_field AND NOT skip_search THEN
+ EXECUTE $$
+ INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
+ VALUES ($$ ||
+ quote_literal(ind_data.field) || $$, $$ ||
+ quote_literal(ind_data.source) || $$, $$ ||
+ quote_literal(ind_data.value) ||
+ $$);$$;
+ END IF;
+
+ END LOOP;
+
+ IF NOT skip_search THEN
+ PERFORM metabib.update_combined_index_vectors(bib_id);
+ END IF;
+
+ RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+DROP FUNCTION IF EXISTS evergreen.oils_tsearch2() CASCADE;
+DROP FUNCTION IF EXISTS public.oils_tsearch2() CASCADE;
+
+CREATE OR REPLACE FUNCTION public.oils_tsearch2 () RETURNS TRIGGER AS $$
+DECLARE
+ normalizer RECORD;
+ value TEXT := '';
+ temp_vector TEXT := '';
+ ts_rec RECORD;
+ cur_weight "char";
+BEGIN
+ value := NEW.value;
+ NEW.index_vector = ''::tsvector;
+
+ IF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
+ FOR normalizer IN
+ SELECT n.func AS func,
+ n.param_count AS param_count,
+ m.params AS params
+ FROM config.index_normalizer n
+ JOIN config.metabib_field_index_norm_map m ON (m.norm = n.id)
+ WHERE field = NEW.field
+ ORDER BY m.pos LOOP
+ EXECUTE 'SELECT ' || normalizer.func || '(' ||
+ quote_literal( value ) ||
+ CASE
+ WHEN normalizer.param_count > 0
+ THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
+ ELSE ''
+ END ||
+ ')' INTO value;
+
+ END LOOP;
+ NEW.value = value;
+ END IF;
+
+ IF TG_TABLE_NAME::TEXT ~ 'browse_entry$' THEN
+ value := ARRAY_TO_STRING(
+ evergreen.regexp_split_to_array(value, E'\\W+'), ' '
+ );
+ value := public.search_normalize(value);
+ NEW.index_vector = to_tsvector(TG_ARGV[0]::regconfig, value);
+ ELSIF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
+ FOR ts_rec IN
+ SELECT ts_config, index_weight
+ FROM config.metabib_class_ts_map
+ WHERE field_class = TG_ARGV[0]
+ AND index_lang IS NULL OR EXISTS (SELECT 1 FROM metabib.record_attr WHERE id = NEW.source AND index_lang IN(attrs->'item_lang',attrs->'language'))
+ AND always OR NOT EXISTS (SELECT 1 FROM config.metabib_field_ts_map WHERE metabib_field = NEW.field)
+ UNION
+ SELECT ts_config, index_weight
+ FROM config.metabib_field_ts_map
+ WHERE metabib_field = NEW.field
+ AND index_lang IS NULL OR EXISTS (SELECT 1 FROM metabib.record_attr WHERE id = NEW.source AND index_lang IN(attrs->'item_lang',attrs->'language'))
+ ORDER BY index_weight ASC
+ LOOP
+ IF cur_weight IS NOT NULL AND cur_weight != ts_rec.index_weight THEN
+ NEW.index_vector = NEW.index_vector || setweight(temp_vector::tsvector,cur_weight);
+ temp_vector = '';
+ END IF;
+ cur_weight = ts_rec.index_weight;
+ SELECT INTO temp_vector temp_vector || ' ' || to_tsvector(ts_rec.ts_config::regconfig, value)::TEXT;
+ END LOOP;
+ NEW.index_vector = NEW.index_vector || setweight(temp_vector::tsvector,cur_weight);
+ ELSE
+ NEW.index_vector = to_tsvector(TG_ARGV[0]::regconfig, value);
+ END IF;
+
+ RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER authority_full_rec_fti_trigger
+ BEFORE UPDATE OR INSERT ON authority.full_rec
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('keyword');
+
+CREATE TRIGGER authority_simple_heading_fti_trigger
+ BEFORE UPDATE OR INSERT ON authority.simple_heading
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('keyword');
+
+CREATE TRIGGER metabib_identifier_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.identifier_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('identifier');
+
+CREATE TRIGGER metabib_title_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.title_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('title');
+
+CREATE TRIGGER metabib_author_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.author_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('author');
+
+CREATE TRIGGER metabib_subject_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.subject_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('subject');
+
+CREATE TRIGGER metabib_keyword_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.keyword_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('keyword');
+
+CREATE TRIGGER metabib_series_field_entry_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.series_field_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('series');
+
+CREATE TRIGGER metabib_browse_entry_fti_trigger
+ BEFORE INSERT OR UPDATE ON metabib.browse_entry
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('keyword');
+
+CREATE TRIGGER metabib_full_rec_fti_trigger
+ BEFORE UPDATE OR INSERT ON metabib.real_full_rec
+ FOR EACH ROW EXECUTE PROCEDURE oils_tsearch2('default');
+
+INSERT INTO config.ts_config_list(id, name) VALUES
+ ('simple','Non-Stemmed Simple'),
+ ('danish_nostop','Danish Stemmed'),
+ ('dutch_nostop','Dutch Stemmed'),
+ ('english_nostop','English Stemmed'),
+ ('finnish_nostop','Finnish Stemmed'),
+ ('french_nostop','French Stemmed'),
+ ('german_nostop','German Stemmed'),
+ ('hungarian_nostop','Hungarian Stemmed'),
+ ('italian_nostop','Italian Stemmed'),
+ ('norwegian_nostop','Norwegian Stemmed'),
+ ('portuguese_nostop','Portuguese Stemmed'),
+ ('romanian_nostop','Romanian Stemmed'),
+ ('russian_nostop','Russian Stemmed'),
+ ('spanish_nostop','Spanish Stemmed'),
+ ('swedish_nostop','Swedish Stemmed'),
+ ('turkish_nostop','Turkish Stemmed');
+
+INSERT INTO config.metabib_class_ts_map(field_class, ts_config, index_weight, always) VALUES
+ ('keyword','simple','A',true),
+ ('keyword','english_nostop','C',true),
+ ('title','simple','A',true),
+ ('title','english_nostop','C',true),
+ ('author','simple','A',true),
+ ('author','english_nostop','C',true),
+ ('series','simple','A',true),
+ ('series','english_nostop','C',true),
+ ('subject','simple','A',true),
+ ('subject','english_nostop','C',true),
+ ('identifier','simple','A',true);
+
+CREATE OR REPLACE FUNCTION evergreen.rel_bump(terms TEXT[], value TEXT, bumps TEXT[], mults NUMERIC[]) RETURNS NUMERIC AS
+$BODY$
+use strict;
+my ($terms,$value,$bumps,$mults) = @_;
+
+my $retval = 1;
+
+for (my $id = 0; $id < @$bumps; $id++) {
+ if ($bumps->[$id] eq 'first_word') {
+ $retval *= $mults->[$id] if ($value =~ /^$terms->[0]/);
+ } elsif ($bumps->[$id] eq 'full_match') {
+ my $fullmatch = join(' ', @$terms);
+ $retval *= $mults->[$id] if ($value =~ /^$fullmatch$/);
+ } elsif ($bumps->[$id] eq 'word_order') {
+ my $wordorder = join('.*', @$terms);
+ $retval *= $mults->[$id] if ($value =~ /$wordorder/);
+ }
+}
+return $retval;
+$BODY$ LANGUAGE plperlu IMMUTABLE STRICT COST 100;
+
+UPDATE metabib.identifier_field_entry set value = value;
+UPDATE metabib.title_field_entry set value = value;
+UPDATE metabib.author_field_entry set value = value;
+UPDATE metabib.subject_field_entry set value = value;
+UPDATE metabib.keyword_field_entry set value = value;
+UPDATE metabib.series_field_entry set value = value;
+
+SELECT metabib.update_combined_index_vectors(id)
+ FROM biblio.record_entry
+ WHERE NOT deleted;
+
+COMMIT;
commit b3a485dd49266d7e6d26bf9c25d7fef9cdced164
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Dec 18 10:38:24 2012 -0500
Add config interfaces
For metabib class, class ts maps, field, and field ts maps.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/templates/conify/global/config/metabib_class.tt2 b/Open-ILS/src/templates/conify/global/config/metabib_class.tt2
new file mode 100644
index 0000000..ee2fe0c
--- /dev/null
+++ b/Open-ILS/src/templates/conify/global/config/metabib_class.tt2
@@ -0,0 +1,25 @@
+[% WRAPPER base.tt2 %]
+[% ctx.page_title = l('MARC Search Classes') %]
+<div dojoType="dijit.layout.ContentPane" layoutAlign="client">
+ <div dojoType="dijit.layout.ContentPane" layoutAlign="top" class='oils-header-panel'>
+ <div>[% l('Metabib Class') %]</div>
+ </div>
+ <div>
+ <table jsId="mbClassGrid"
+ dojoType="openils.widget.AutoGrid"
+ fieldOrder="['name', 'label', 'buoyant', 'restrict', 'a_weight', 'b_weight', 'c_weight', 'd_weight']"
+ query="{name: '*'}"
+ fmClass='cmc'
+ autoHeight='true'
+ editOnEnter='true'>
+ </table>
+</div>
+
+<script type="text/javascript">
+ dojo.require('openils.Util');
+ dojo.require('openils.widget.AutoGrid');
+ openils.Util.addOnLoad( function() { mbClassGrid.loadAll(); } );
+</script>
+[% END %]
+
+
diff --git a/Open-ILS/src/templates/conify/global/config/metabib_class_ts_map.tt2 b/Open-ILS/src/templates/conify/global/config/metabib_class_ts_map.tt2
new file mode 100644
index 0000000..4874ddf
--- /dev/null
+++ b/Open-ILS/src/templates/conify/global/config/metabib_class_ts_map.tt2
@@ -0,0 +1,29 @@
+[% WRAPPER base.tt2 %]
+[% ctx.page_title = l('MARC Search Class FTS Config Maps') %]
+<div dojoType="dijit.layout.ContentPane" layoutAlign="client">
+ <div dojoType="dijit.layout.ContentPane" layoutAlign="top" class='oils-header-panel'>
+ <div>[% l('Metabib Class FTS Config Maps') %]</div>
+ <div>
+ <button dojoType='dijit.form.Button' onClick='mbClassTSMGrid.showCreateDialog()'>[% l('New FTS Map') %]</button>
+ <button dojoType='dijit.form.Button' onClick='mbClassTSMGrid.deleteSelected()'>[% l('Delete Selected') %]</button>
+ </div>
+ </div>
+ <div>
+ <table jsId="mbClassTSMGrid"
+ dojoType="openils.widget.AutoGrid"
+ fieldOrder="['id', 'field_class', 'ts_config', 'active', 'index_weight', 'index_lang', 'search_lang', 'always']"
+ query="{id: '*'}"
+ fmClass='cmcts'
+ autoHeight='true'
+ editOnEnter='true'>
+ </table>
+</div>
+
+<script type="text/javascript">
+ dojo.require('openils.Util');
+ dojo.require('openils.widget.AutoGrid');
+ openils.Util.addOnLoad( function() { mbClassTSMGrid.loadAll(); } );
+</script>
+[% END %]
+
+
diff --git a/Open-ILS/src/templates/conify/global/config/metabib_field_ts_map.tt2 b/Open-ILS/src/templates/conify/global/config/metabib_field_ts_map.tt2
new file mode 100644
index 0000000..1d79409
--- /dev/null
+++ b/Open-ILS/src/templates/conify/global/config/metabib_field_ts_map.tt2
@@ -0,0 +1,29 @@
+[% WRAPPER base.tt2 %]
+[% ctx.page_title = l('MARC Search and Facet Fields FTS Configs') %]
+<div dojoType="dijit.layout.ContentPane" layoutAlign="client">
+ <div dojoType="dijit.layout.ContentPane" layoutAlign="top" class='oils-header-panel'>
+ <div>[% l('Metabib Field FTS Configs') %]</div>
+ <div>
+ <button dojoType='dijit.form.Button' onClick='mbFieldFTSGrid.showCreateDialog()'>[% l('New FTS Config') %]</button>
+ <button dojoType='dijit.form.Button' onClick='mbFieldFTSGrid.deleteSelected()'>[% l('Delete Selected') %]</button>
+ </div>
+ </div>
+ <div>
+ <table jsId="mbFieldFTSGrid"
+ dojoType="openils.widget.AutoGrid"
+ fieldOrder="['id', 'metabib_field', 'ts_config', 'active', 'index_weight', 'index_lang', 'search_lang']"
+ query="{field: '*'}"
+ fmClass='cmfts'
+ autoHeight='true'
+ editOnEnter='true'>
+ </table>
+</div>
+
+<script type="text/javascript">
+ dojo.require('openils.Util');
+ dojo.require('openils.widget.AutoGrid');
+ openils.Util.addOnLoad( function() { mbFieldFTSGrid.loadAll(); } );
+</script>
+[% END %]
+
+
diff --git a/Open-ILS/web/opac/locale/en-US/lang.dtd b/Open-ILS/web/opac/locale/en-US/lang.dtd
index f5880ff..e0cb546 100644
--- a/Open-ILS/web/opac/locale/en-US/lang.dtd
+++ b/Open-ILS/web/opac/locale/en-US/lang.dtd
@@ -758,7 +758,10 @@
<!ENTITY staff.main.menu.admin.server_admin.conify.copy_status.label "Copy Statuses">
<!ENTITY staff.main.menu.admin.server_admin.conify.marc_record_attrs.label "MARC Record Attributes">
<!ENTITY staff.main.menu.admin.server_admin.conify.coded_value_maps.label "MARC Coded Value Maps">
+<!ENTITY staff.main.menu.admin.server_admin.conify.metabib_class.label "MARC Search/Facet Classes">
+<!ENTITY staff.main.menu.admin.server_admin.conify.metabib_class_ts_map.label "MARC Search/Facet Class FTS Maps">
<!ENTITY staff.main.menu.admin.server_admin.conify.metabib_field.label "MARC Search/Facet Fields">
+<!ENTITY staff.main.menu.admin.server_admin.conify.metabib_field_ts_map.label "MARC Search/Facet Field FTS Maps">
<!ENTITY staff.main.menu.admin.server_admin.conify.acn_prefix.label "Call Number Prefixes">
<!ENTITY staff.main.menu.admin.server_admin.conify.acn_suffix.label "Call Number Suffixes">
<!ENTITY staff.main.menu.admin.server_admin.conify.billing_type.label "Billing Types">
diff --git a/Open-ILS/xul/staff_client/chrome/content/main/menu.js b/Open-ILS/xul/staff_client/chrome/content/main/menu.js
index 322ca8a..29f0901 100644
--- a/Open-ILS/xul/staff_client/chrome/content/main/menu.js
+++ b/Open-ILS/xul/staff_client/chrome/content/main/menu.js
@@ -913,10 +913,22 @@ main.menu.prototype = {
['oncommand'],
function(event) { open_eg_web_page('conify/global/config/coded_value_map', null, event); }
],
+ 'cmd_server_admin_metabib_class' : [
+ ['oncommand'],
+ function(event) { open_eg_web_page('conify/global/config/metabib_class', null, event); }
+ ],
+ 'cmd_server_admin_metabib_class_ts_map' : [
+ ['oncommand'],
+ function(event) { open_eg_web_page('conify/global/config/metabib_class_ts_map', null, event); }
+ ],
'cmd_server_admin_metabib_field' : [
['oncommand'],
function(event) { open_eg_web_page('conify/global/config/metabib_field', null, event); }
],
+ 'cmd_server_admin_metabib_field_ts_map' : [
+ ['oncommand'],
+ function(event) { open_eg_web_page('conify/global/config/metabib_field_ts_map', null, event); }
+ ],
'cmd_server_admin_acn_prefix' : [
['oncommand'],
function(event) { open_eg_web_page('conify/global/config/acn_prefix', null, event); }
diff --git a/Open-ILS/xul/staff_client/chrome/content/main/menu_frame_menus.xul b/Open-ILS/xul/staff_client/chrome/content/main/menu_frame_menus.xul
index 7fe88a2..8967f9c 100644
--- a/Open-ILS/xul/staff_client/chrome/content/main/menu_frame_menus.xul
+++ b/Open-ILS/xul/staff_client/chrome/content/main/menu_frame_menus.xul
@@ -215,9 +215,15 @@
<command id="cmd_server_admin_coded_value_map"
perm="ADMIN_CODED_VALUE"
/>
+ <command id="cmd_server_admin_metabib_class"
+ perm="UPDATE_METABIB_CLASS"/>
+ <command id="cmd_server_admin_metabib_class_ts_map"
+ perm="ADMIN_INDEX_NORMALIZER"/>
<command id="cmd_server_admin_metabib_field"
perm="CREATE_METABIB_FIELD DELETE_METABIB_FIELD UPDATE_METABIB_FIELD"
/>
+ <command id="cmd_server_admin_metabib_field_ts_map"
+ perm="ADMIN_INDEX_NORMALIZER"/>
<command id="cmd_server_admin_billing_type"
perm="CREATE_BILLING_TYPE DELETE_BILLING_TYPE UPDATE_BILLING_TYPE"
/>
@@ -600,7 +606,10 @@
<menuitem label="&staff.main.menu.admin.server_admin.conify.acn_suffix.label;" command="cmd_server_admin_acn_suffix"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.marc_record_attrs.label;" command="cmd_server_admin_marc_code"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.coded_value_maps.label;" command="cmd_server_admin_coded_value_map"/>
+ <menuitem label="&staff.main.menu.admin.server_admin.conify.metabib_class.label;" command="cmd_server_admin_metabib_class"/>
+ <menuitem label="&staff.main.menu.admin.server_admin.conify.metabib_class_ts_map.label;" command="cmd_server_admin_metabib_class_ts_map"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.metabib_field.label;" command="cmd_server_admin_metabib_field"/>
+ <menuitem label="&staff.main.menu.admin.server_admin.conify.metabib_field_ts_map.label;" command="cmd_server_admin_metabib_field_ts_map"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.billing_type.label;" command="cmd_server_admin_billing_type"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.sms_carrier.label;" command="cmd_server_admin_sms_carrier"/>
<menuitem label="&staff.main.menu.admin.server_admin.conify.z3950_source.label;" command="cmd_server_admin_z39_source"/>
commit e1149c69cf994f9f04441056b7e0b0538ac3d0ce
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Thu Oct 11 11:43:25 2012 -0400
QueryParser Driver: Much work
Switch to configurable fts configs
Add "combined" index vectors
Add word boundary checks for phrase searches
Use combined rel_bump function
And probably more I forgot about
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml
index 1510df1..ce64b5f 100644
--- a/Open-ILS/examples/fm_IDL.xml
+++ b/Open-ILS/examples/fm_IDL.xml
@@ -2311,6 +2311,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
<field reporter:label="Label" name="label" reporter:datatype="text" oils_persist:i18n="true"/>
<field reporter:label="Buoyant?" name="buoyant" reporter:datatype="bool" />
<field reporter:label="Restrict?" name="restrict" reporter:datatype="bool" />
+ <field reporter:label="A Weight" name="a_weight" reporter:datatype="float" />
+ <field reporter:label="B Weight" name="b_weight" reporter:datatype="float" />
+ <field reporter:label="C Weight" name="c_weight" reporter:datatype="float" />
+ <field reporter:label="D Weight" name="d_weight" reporter:datatype="float" />
<field reporter:label="Fields" name="fields" reporter:datatype="link" oils_persist:virtual="true"/>
</fields>
<links>
@@ -9821,6 +9825,68 @@ SELECT usr,
</permacrud>
</class>
+ <class id="ctcl" controller="open-ils.cstore open-ils.pcrud" oils_obj:fieldmapper="config::ts_config_list" oils_persist:tablename="config.ts_config_list" reporter:label="Text Search Configs">
+ <fields oils_persist:primary="id">
+ <field reporter:label="Text Search Config" reporter:selector="name" reporter:datatype="text" name="id"/>
+ <field reporter:label="Text Search Config Name" reporter:datatype="text" name="name"/>
+ </fields>
+ <permacrud xmlns="http://open-ils.org/spec/opensrf/IDL/permacrud/v1">
+ <actions>
+ <retrieve/>
+ </actions>
+ </permacrud>
+ </class>
+
+ <class id="cmcts" controller="open-ils.cstore open-ils.pcrud" oils_obj:fieldmapper="config::metabib_class_ts_map" oils_persist:tablename="config.metabib_class_ts_map" reporter:label="Metabib Class TS Map">
+ <fields oils_persist:primary="id" oils_persist:sequence="config.metabib_class_ts_map_id_seq">
+ <field reporter:label="Map ID" reporter:datatype="id" name="id"/>
+ <field reporter:label="Metabib Class" reporter:datatype="text" name="field_class"/>
+ <field reporter:label="Text Search Config" reporter:datatype="text" name="ts_config"/>
+ <field reporter:label="Active?" reporter:datatype="bool" name="active"/>
+ <field reporter:label="Index Weight" reporter:datatype="text" name="index_weight"/>
+ <field reporter:label="Index Language" reporter:datatype="text" name="index_lang"/>
+ <field reporter:label="Search Language" reporter:datatype="text" name="search_lang"/>
+ <field reporter:label="Always Apply?" reporter:datatype="bool" name="always"/>
+ </fields>
+ <links>
+ <link field="field_class" reltype="has_a" key="name" map="" class="cmc"/>
+ <link field="ts_config" reltype="has_a" key="id" map="" class="ctcl"/>
+ </links>
+ <permacrud xmlns="http://open-ils.org/spec/opensrf/IDL/permacrud/v1">
+ <actions>
+ <create permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ <retrieve/>
+ <update permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ <delete permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ </actions>
+ </permacrud>
+ </class>
+
+ <class id="cmfts" controller="open-ils.cstore open-ils.pcrud" oils_obj:fieldmapper="config::metabib_field_ts_map" oils_persist:tablename="config.metabib_field_ts_map" reporter:label="Metabib Field TS Map">
+ <fields oils_persist:primary="id" oils_persist:sequence="config.metabib_field_ts_map_id_seq">
+ <field reporter:label="Map ID" reporter:datatype="id" name="id"/>
+ <field reporter:label="Metabib Field" reporter:datatype="text" name="metabib_field"/>
+ <field reporter:label="Text Search Config" reporter:datatype="text" name="ts_config"/>
+ <field reporter:label="Active?" reporter:datatype="bool" name="active"/>
+ <field reporter:label="Index Weight" reporter:datatype="text" name="index_weight"/>
+ <field reporter:label="Index Language" reporter:datatype="text" name="index_lang"/>
+ <field reporter:label="Search Language" reporter:datatype="text" name="search_lang"/>
+ </fields>
+ <links>
+ <link field="metabib_field" reltype="has_a" key="id" map="" class="cmf"/>
+ <link field="ts_config" reltype="has_a" key="id" map="" class="ctcl"/>
+ </links>
+ <permacrud xmlns="http://open-ils.org/spec/opensrf/IDL/permacrud/v1">
+ <actions>
+ <create permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ <retrieve/>
+ <update permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ <delete permission="ADMIN_INDEX_NORMALIZER" global_required="true"/>
+ </actions>
+ </permacrud>
+ </class>
+
+
<!-- ********************************************************************************************************************* -->
<!-- What follows is a set of example extensions that are useful for PINES. Comment out or remove if you don't want them. -->
<!-- ********************************************************************************************************************* -->
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 9afb754..b5fc794 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -98,14 +98,19 @@ sub quote_value {
sub quote_phrase_value {
my $self = shift;
my $value = shift;
-
- my $left_anchored = $value =~ m/^\^/;
- my $right_anchored = $value =~ m/\$$/;
- $value =~ s/\^// if $left_anchored;
- $value =~ s/\$$// if $right_anchored;
+ my $wb = shift;
+
+ my $left_anchored = '';
+ my $right_anchored = '';
+ $left_anchored = $1 if $value =~ m/^([*\^])/;
+ $right_anchored = $1 if $value =~ m/([*\$])$/;
+ $value =~ s/^[*\^]// if $left_anchored;
+ $value =~ s/[*\$]$// if $right_anchored;
$value = quotemeta($value);
- $value = '^' . $value if $left_anchored;
- $value = "$value\$" if $right_anchored;
+ $value = '^' . $value if $left_anchored eq '^';
+ $value = "$value\$" if $right_anchored eq '$';
+ $value = '[[:<:]]' . $value if $wb && !$left_anchored;
+ $value .= '[[:>:]]' if $wb && !$right_anchored;
return $self->quote_value($value);
}
@@ -297,6 +302,78 @@ sub add_relevance_bump {
return { $class => { $field => { $type => { multiplier => $multiplier, active => $active } } } };
}
+sub search_class_weights {
+ my $self = shift;
+ my $class = shift;
+ my $a_weight = shift;
+ my $b_weight = shift;
+ my $c_weight = shift;
+ my $d_weight = shift;
+
+ $self->custom_data->{class_weights} ||= {};
+ # Note: This reverses the A-D order, putting D first, because that is how the call actually works in PG
+ $self->custom_data->{class_weights}->{$class} ||= [0.1, 0.2, 0.4, 1.0];
+ $self->custom_data->{class_weights}->{$class} = [$d_weight, $c_weight, $b_weight, $a_weight] if $a_weight;
+ return $self->custom_data->{class_weights}->{$class};
+}
+
+sub class_ts_config {
+ my $self = shift;
+ my $class = shift;
+ my $lang = shift || 'DEFAULT';
+ my $always = shift;
+ my $ts_config = shift;
+
+ $self->custom_data->{class_ts_config} ||= {};
+ $self->custom_data->{class_ts_config}->{$class} ||= {};
+ $self->custom_data->{class_ts_config}->{$class}->{$lang} ||= {};
+ $self->custom_data->{class_ts_config}->{$class}->{$lang}->{normal} ||= [];
+ $self->custom_data->{class_ts_config}->{$class}->{$lang}->{always} ||= [];
+ $self->custom_data->{class_ts_config}->{$class}->{'DEFAULT'} ||= {};
+ $self->custom_data->{class_ts_config}->{$class}->{'DEFAULT'}->{normal} ||= [];
+ $self->custom_data->{class_ts_config}->{$class}->{'DEFAULT'}->{always} ||= [];
+
+ if ($ts_config) {
+ push @{$self->custom_data->{class_ts_config}->{$class}->{$lang}->{normal}}, $ts_config unless $always;
+ push @{$self->custom_data->{class_ts_config}->{$class}->{$lang}->{always}}, $ts_config if $always;
+ }
+
+ my $return = [];
+ push @$return, @{$self->custom_data->{class_ts_config}->{$class}->{$lang}->{always}};
+ push @$return, @{$self->custom_data->{class_ts_config}->{$class}->{$lang}->{normal}} unless $always;
+ if($lang ne 'DEFAULT') {
+ push @$return, @{$self->custom_data->{class_ts_config}->{$class}->{'DEFAULT'}->{always}};
+ push @$return, @{$self->custom_data->{class_ts_config}->{$class}->{'DEFAULT'}->{normal}} unless $always;
+ }
+ return $return;
+}
+
+sub field_ts_config {
+ my $self = shift;
+ my $class = shift;
+ my $field = shift;
+ my $lang = shift || 'DEFAULT';
+ my $ts_config = shift;
+
+ $self->custom_data->{field_ts_config} ||= {};
+ $self->custom_data->{field_ts_config}->{$class} ||= {};
+ $self->custom_data->{field_ts_config}->{$class}->{$field} ||= {};
+ $self->custom_data->{field_ts_config}->{$class}->{$field}->{$lang} ||= [];
+ $self->custom_data->{field_ts_config}->{$class}->{$field}->{'DEFAULT'} ||= [];
+
+ if ($ts_config) {
+ push @{$self->custom_data->{field_ts_config}->{$class}->{$field}->{$lang}}, $ts_config;
+ }
+
+ my $return = [];
+ push @$return, @{$self->custom_data->{field_ts_config}->{$class}->{$field}->{$lang}};
+ if($lang ne 'DEFAULT') {
+ push @$return, @{$self->custom_data->{field_ts_config}->{$class}->{$field}->{'DEFAULT'}};
+ }
+ # Make it easy on us: Grab any "always" for the class here. If we have none we grab them all.
+ push @$return, @{$self->class_ts_config($class, $lang, scalar(@$return))};
+ return $return;
+}
sub initialize_search_field_id_map {
my $self = shift;
@@ -365,6 +442,36 @@ sub initialize_filter_normalizers {
}
}
+sub initialize_class_weights {
+ my $self = shift;
+ my $classes = shift;
+
+ for my $search_class (@$classes) {
+ __PACKAGE__->search_class_weights( $search_class->name, $search_class->a_weight, $search_class->b_weight, $search_class->c_weight, $search_class->d_weight );
+ }
+}
+
+sub initialize_class_ts_config {
+ my $self = shift;
+ my $class_entries = shift;
+
+ for my $search_class_entry (@$class_entries) {
+ __PACKAGE__->class_ts_config($search_class_entry->field_class,$search_class_entry->search_lang,$U->is_true($search_class_entry->always),$search_class_entry->ts_config);
+ }
+}
+
+sub initialize_field_ts_config {
+ my $self = shift;
+ my $field_entries = shift;
+ my $field_objects = shift;
+ my %field_hash = map { $_->id => $_ } @$field_objects;
+
+ for my $search_field_entry (@$field_entries) {
+ my $field_object = $field_hash{$search_field_entry->metabib_field};
+ __PACKAGE__->field_ts_config($field_object->field_class,$field_object->name,$search_field_entry->search_lang,$search_field_entry->ts_config);
+ }
+}
+
our $_complete = 0;
sub initialization_complete {
return $_complete;
@@ -406,6 +513,15 @@ sub initialize {
$self->initialize_filter_normalizers( $args{config_record_attr_index_norm_map} )
if ($args{config_record_attr_index_norm_map});
+ $self->initialize_search_class_weights( $args{config_metabib_class} )
+ if ($args{config_metabib_class});
+
+ $self->initialize_class_ts_config( $args{config_metabib_class_ts_map} )
+ if ($args{config_metabib_class_ts_map});
+
+ $self->initialize_field_ts_config( $args{config_metabib_field_ts_map}, $args{config_metabib_field} )
+ if ($args{config_metabib_field_ts_map} && $args{config_metabib_field});
+
$_complete = 1 if (
$args{config_metabib_field_index_norm_map} &&
$args{search_relevance_adjustment} &&
@@ -467,6 +583,27 @@ sub TEST_SETUP {
__PACKAGE__->add_relevance_bump( keyword => keyword => first_word => 1 );
__PACKAGE__->add_relevance_bump( keyword => keyword => full_match => 1 );
+ __PACKAGE__->class_ts_config( 'series', undef, 1, 'english_nostop' );
+ __PACKAGE__->class_ts_config( 'title', undef, 1, 'english_nostop' );
+ __PACKAGE__->class_ts_config( 'author', undef, 1, 'english_nostop' );
+ __PACKAGE__->class_ts_config( 'subject', undef, 1, 'english_nostop' );
+ __PACKAGE__->class_ts_config( 'keyword', undef, 1, 'english_nostop' );
+ __PACKAGE__->class_ts_config( 'series', undef, 1, 'simple' );
+ __PACKAGE__->class_ts_config( 'title', undef, 1, 'simple' );
+ __PACKAGE__->class_ts_config( 'author', undef, 1, 'simple' );
+ __PACKAGE__->class_ts_config( 'subject', undef, 1, 'simple' );
+ __PACKAGE__->class_ts_config( 'keyword', undef, 1, 'simple' );
+
+ # French! To test language limiters
+ __PACKAGE__->class_ts_config( 'series', 'fre', 1, 'french_nostop' );
+ __PACKAGE__->class_ts_config( 'title', 'fre', 1, 'french_nostop' );
+ __PACKAGE__->class_ts_config( 'author', 'fre', 1, 'french_nostop' );
+ __PACKAGE__->class_ts_config( 'subject', 'fre', 1, 'french_nostop' );
+ __PACKAGE__->class_ts_config( 'keyword', 'fre', 1, 'french_nostop' );
+
+ # Not a default config by any means, but good for some testing
+ __PACKAGE__->field_ts_config( 'author', 'personal', 'eng', 'english' );
+ __PACKAGE__->field_ts_config( 'author', 'personal', 'fre', 'french' );
__PACKAGE__->add_search_class_alias( keyword => 'kw' );
__PACKAGE__->add_search_class_alias( title => 'ti' );
@@ -831,29 +968,6 @@ SQL
}
-
-sub rel_bump {
- my $self = shift;
- my $node = shift;
- my $bump = shift;
- my $multiplier = shift;
-
- my $only_atoms = $node->only_real_atoms;
- return '' if (!@$only_atoms);
-
- if ($bump eq 'first_word') {
- return "/* first_word */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||search_normalize(".$self->QueryParser->quote_phrase_value($only_atoms->[0]->content)."))), FALSE )::INT * $multiplier, 1)";
- } elsif ($bump eq 'full_match') {
- return "/* full_match */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||".
- join( "||' '||", map { "search_normalize(".$self->QueryParser->quote_phrase_value($_->content).")" } @$only_atoms )."||'\$')), FALSE )::INT * $multiplier, 1)";
- } elsif ($bump eq 'word_order') {
- return "/* word_order */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ (".
- join( "||'.*'||", map { "search_normalize(".$self->QueryParser->quote_phrase_value($_->content).")" } @$only_atoms ).")), FALSE )::INT * $multiplier, 1)";
- }
-
- return '';
-}
-
sub flatten {
my $self = shift;
@@ -874,21 +988,27 @@ sub flatten {
}
my $table = $node->table;
+ my $ctable = $node->combined_table;
my $talias = $node->table_alias;
my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 0.0)";
$from .= "\n" . ${spc} x 4 ."LEFT JOIN (\n"
- . ${spc} x 5 . "SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq /* search */\n"
+ . ${spc} x 5 . "SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq, ${talias}_xq.tsq_rank /* search */\n"
. ${spc} x 6 . "FROM $table AS fe";
$from .= "\n" . ${spc} x 7 . "JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
if ($node->dummy_count < @{$node->only_atoms} ) {
$with .= ",\n " if $with;
- $with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq )";
- $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
+ $with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq,". $node->tsquery_rank ." AS tsq_rank )";
+ $from .= "\n" . ${spc} x 6 . "JOIN $ctable AS com ON (com.record = fe.source)";
+ if (@{$node->fields} > 0) {
+ $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (com.index_vector @@ ${talias}_xq.tsq_rank AND fe.index_vector @@ ${talias}_xq.tsq)";
+ } else {
+ $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (com.index_vector @@ ${talias}_xq.tsq)";
+ }
} else {
- $from .= "\n" . ${spc} x 6 . ", (SELECT NULL::tsquery AS tsq ) AS ${talias}_xq";
+ $from .= "\n" . ${spc} x 6 . ", (SELECT NULL::tsquery AS tsq, NULL:tsquery AS tsq_rank ) AS ${talias}_xq";
}
my @bump_fields;
@@ -915,6 +1035,8 @@ sub flatten {
my %used_bumps;
+ my @bumps;
+ my @bumpmults;
for my $field ( @bump_fields ) {
my $bumps = $self->QueryParser->find_relevance_bumps( $node->classname => $field );
for my $b (keys %$bumps) {
@@ -923,24 +1045,31 @@ sub flatten {
$used_bumps{$b} = 1;
next if ($$bumps{$b}{multiplier} == 1); # optimization to remove unneeded bumps
-
- my $bump_case = $self->rel_bump( $node, $b, $$bumps{$b}{multiplier} );
- $node_rank .= "\n" . ${spc} x 5 . "* " . $bump_case if ($bump_case);
+ push @bumps, $b;
+ push @bumpmults, $$bumps{$b}{multiplier};
}
}
+ if(scalar @bumps > 0 && scalar @{$node->only_positive_atoms} > 0) {
+ # Note: Previous rank function used search_normalize outright. Duplicating that here.
+ $node_rank .= "\n" . ${spc} x 5 . "* evergreen.rel_bump(('{' || search_normalize(";
+ $node_rank .= join(") || ',' || search_normalize(",map { $self->QueryParser->quote_phrase_value($_->content) } @{$node->only_positive_atoms});
+ $node_rank .= ") || '}')::TEXT[], " . $node->table_alias . ".value, '{" . join(",", at bumps) . "}'::TEXT[], '{" . join(",", at bumpmults) . "}'::NUMERIC[])";
+ }
+
my $NOT = '';
$NOT = 'NOT ' if $node->negate;
$where .= "$NOT(" . $talias . ".id IS NOT NULL";
if (@{$node->phrases}) {
$where .= ' AND ' . join(' AND ', map {
- "${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)
+ "${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_, 1)
} @{$node->phrases});
- }
- for my $atom (@{$node->only_real_atoms}) {
- next unless $atom->{content} && $atom->{content} =~ /(^\^|\$$)/;
- $where .= " AND ${talias}.value ~* ".$self->QueryParser->quote_phrase_value($atom->{content});
+ } else {
+ for my $atom (@{$node->only_real_atoms}) {
+ next unless $atom->{content} && $atom->{content} =~ /(^\^|\$$)/;
+ $where .= " AND ${talias}.value ~* ".$self->QueryParser->quote_phrase_value($atom->{content});
+ }
}
$where .= ')';
@@ -948,7 +1077,6 @@ sub flatten {
} elsif ($node->isa( 'QueryParser::query_plan::facet' )) {
- my $table = $node->table;
my $talias = $node->table_alias;
my @field_ids;
@@ -1211,11 +1339,6 @@ sub classname {
return $classname;
}
-sub table {
- my $self = shift;
- return 'metabib.' . $self->classname . '_field_entry';
-}
-
sub fields {
my $self = shift;
my ($classname, at fields) = split '\|', $self->name;
@@ -1262,6 +1385,30 @@ sub buildSQL {
my $normalizers = $self->node->plan->QueryParser->query_normalizers( $classname );
my $fields = $self->node->fields;
+ my $lang;
+ my $filter = $self->node->plan->find_filter('preferred_language');
+ $lang ||= $filter->args->[0] if ($filter && $filter->args);
+ $lang ||= $self->node->plan->QueryParser->default_preferred_language;
+ my $ts_configs = [];
+
+ if (@{$self->node->phrases}) {
+ # We assume we want 'simple' for phrases. Gives us less to match against later.
+ $ts_configs = ['simple'];
+ } else {
+ if (!@$fields) {
+ $ts_configs = $self->node->plan->QueryParser->class_ts_config($classname, $lang);
+ } else {
+ for my $field (@$fields) {
+ push @$ts_configs, @{$self->node->plan->QueryParser->field_ts_config($classname, $field, $lang)};
+ }
+ }
+ $ts_configs = [keys %{{map { $_ => 1 } @$ts_configs}}];
+ }
+
+ # Assume we want exact if none otherwise provided.
+ # Because we can reasonably expect this to exist
+ $ts_configs = ['simple'] unless (scalar @$ts_configs);
+
$fields = $self->node->plan->QueryParser->search_fields->{$classname} if (!@$fields);
my %norms;
@@ -1288,6 +1435,8 @@ sub buildSQL {
my $prefix = $self->prefix || '';
my $suffix = $self->suffix || '';
+ my $joiner = ' || ';
+ $joiner = ' && ' if $self->prefix eq '!'; # Negative atoms should be "none of the variants" instead of "any of the variants"
$prefix = "'$prefix' ||" if $prefix;
my $suffix_op = '';
@@ -1296,7 +1445,13 @@ sub buildSQL {
$suffix_op = ":$suffix" if $suffix;
$suffix_after = "|| '$suffix_op'" if $suffix;
- $sql = "to_tsquery('$classname', COALESCE(NULLIF($prefix '(' || btrim(regexp_replace($sql,E'(?:\\\\s+|:)','$suffix_op&','g'),'&|') $suffix_after || ')', '()'), ''))";
+ my @sql_set = ();
+ for my $ts_config (@$ts_configs) {
+ push @sql_set, "to_tsquery('$ts_config', COALESCE(NULLIF($prefix '(' || btrim(regexp_replace($sql,E'(?:\\\\s+|:)','$suffix_op&','g'),'&|') $suffix_after || ')', '()'), ''))";
+ }
+
+ $sql = join($joiner, @sql_set);
+ $sql = '(' . $sql . ')' if (scalar(@$ts_configs) > 1);
return $self->sql($sql);
}
@@ -1332,6 +1487,18 @@ sub only_real_atoms {
return \@only_real_atoms;
}
+sub only_positive_atoms {
+ my $self = shift;
+
+ my $atoms = $self->query_atoms;
+ my @only_positive_atoms;
+ for my $a (@$atoms) {
+ push(@only_positive_atoms, $a) if (ref($a) && $a->isa('QueryParser::query_plan::node::atom') && !($a->{dummy}) && ($a->{prefix} ne '!'));
+ }
+
+ return \@only_positive_atoms;
+}
+
sub dummy_count {
my $self = shift;
return $self->{dummy_count};
@@ -1345,6 +1512,14 @@ sub table {
return $self->table( 'metabib.' . $self->classname . '_field_entry' );
}
+sub combined_table {
+ my $self = shift;
+ my $ctable = shift;
+ $self->{ctable} = $ctable if ($ctable);
+ return $self->{ctable} if $self->{ctable};
+ return $self->combined_table( 'metabib.combined_' . $self->classname . '_field_entry' );
+}
+
sub table_alias {
my $self = shift;
my $table_alias = shift;
@@ -1374,8 +1549,21 @@ sub tsquery {
return $self->{tsquery};
}
+sub tsquery_rank {
+ my $self = shift;
+ return $self->{tsquery_rank} if ($self->{tsquery_rank});
+ my @atomlines;
+
+ for my $atom (@{$self->only_positive_atoms}) {
+ push @atomlines, "\n" . ${spc} x 3 . $atom->sql;
+ }
+ $self->{tsquery_rank} = join(' ||', @atomlines);
+ return $self->{tsquery_rank};
+}
+
sub rank {
my $self = shift;
+ return $self->{rank} if ($self->{rank});
my $rank_norm_map = $self->plan->QueryParser->custom_data->{rank_cd_weight_map};
@@ -1384,8 +1572,9 @@ sub rank {
$cover_density += $$rank_norm_map{$norm} if ($self->plan->find_modifier($norm));
}
- return $self->{rank} if ($self->{rank});
- return $self->{rank} = 'ts_rank_cd(' . $self->table_alias . '.index_vector, ' . $self->table_alias . ".tsq, $cover_density)";
+ my $weights = join(', ', @{$self->plan->QueryParser->search_class_weights($self->classname)});
+
+ return $self->{rank} = "ts_rank_cd('{" . $weights . "}', " . $self->table_alias . '.index_vector, ' . $self->table_alias . ".tsq_rank, $cover_density)";
}
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
index 3241bc3..96a70cf 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
@@ -53,6 +53,21 @@ sub _initialize_parser {
'open-ils.cstore.direct.config.record_attr_definition.search.atomic',
{ name => { "!=" => undef } }
)->gather(1),
+ config_metabib_class_ts_map =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_class_ts_map.search.atomic',
+ { active => "t" }
+ )->gather(1),
+ config_metabib_field_ts_map =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_field_ts_map.search.atomic',
+ { active => "t" }
+ )->gather(1),
+ config_metabib_class =>
+ $cstore->request(
+ 'open-ils.cstore.direct.config.metabib_class.search.atomic',
+ { name => { "!=" => undef } }
+ )->gather(1),
);
$cstore->disconnect;
diff --git a/Open-ILS/src/sql/Pg/000.english.pg90.fts-config.sql b/Open-ILS/src/sql/Pg/000.english.pg90.fts-config.sql
deleted file mode 100644
index 7ddce06..0000000
--- a/Open-ILS/src/sql/Pg/000.english.pg90.fts-config.sql
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (C) 2004-2008 Georgia Public Library Service
- * Copyright (C) 2008 Equinox Software, Inc., Laurentian University
- * Mike Rylander <miker at esilibrary.com>
- * Dan Scott <dscott at laurentian.ca>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- */
-
-BEGIN;
-
-SET search_path = public, pg_catalog;
-
-CREATE OR REPLACE FUNCTION oils_tsearch2 () RETURNS TRIGGER AS $$
-BEGIN
- NEW.index_vector = to_tsvector((TG_ARGV[0])::regconfig, NEW.value);
- RETURN NEW;
-END;
-$$ LANGUAGE PLPGSQL;
-
-DROP TEXT SEARCH DICTIONARY IF EXISTS english_nostop CASCADE;
-
-CREATE TEXT SEARCH DICTIONARY english_nostop (TEMPLATE=pg_catalog.snowball, language='english');
-COMMENT ON TEXT SEARCH DICTIONARY english_nostop IS 'English snowball stemmer with no stopwords for ASCII words only.';
-
-CREATE TEXT SEARCH CONFIGURATION title ( COPY = pg_catalog.english );
-ALTER TEXT SEARCH CONFIGURATION title ALTER MAPPING FOR word, hword, hword_part WITH pg_catalog.simple;
-ALTER TEXT SEARCH CONFIGURATION title ALTER MAPPING FOR asciiword, asciihword, hword_asciipart WITH english_nostop;
-CREATE TEXT SEARCH CONFIGURATION author ( COPY = title );
-CREATE TEXT SEARCH CONFIGURATION subject ( COPY = title );
-CREATE TEXT SEARCH CONFIGURATION keyword ( COPY = title );
-CREATE TEXT SEARCH CONFIGURATION identifier ( COPY = title );
-CREATE TEXT SEARCH CONFIGURATION series ( COPY = title );
-CREATE TEXT SEARCH CONFIGURATION "default" ( COPY = title );
-
-COMMIT;
diff --git a/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql b/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql
deleted file mode 120000
index fd3fe58..0000000
--- a/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql
+++ /dev/null
@@ -1 +0,0 @@
-000.english.pg90.fts-config.sql
\ No newline at end of file
diff --git a/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql b/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql
new file mode 100644
index 0000000..0419aa7
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2004-2008 Georgia Public Library Service
+ * Copyright (C) 2008 Equinox Software, Inc., Laurentian University
+ * Mike Rylander <miker at esilibrary.com>
+ * Dan Scott <dscott at laurentian.ca>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+BEGIN;
+
+SET search_path = public, pg_catalog;
+
+CREATE OR REPLACE FUNCTION oils_tsearch2 () RETURNS TRIGGER AS $$
+BEGIN
+ NEW.index_vector = to_tsvector((TG_ARGV[0])::regconfig, NEW.value);
+ RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+DO $$
+DECLARE
+lang TEXT;
+BEGIN
+FOR lang IN SELECT substring(pptsd.dictname from '(.*)_stem$') AS lang FROM pg_catalog.pg_ts_dict pptsd JOIN pg_catalog.pg_namespace ppn ON ppn.oid = pptsd.dictnamespace
+WHERE ppn.nspname = 'pg_catalog' AND pptsd.dictname LIKE '%_stem' LOOP
+RAISE NOTICE 'FOUND LANGUAGE %', lang;
+
+EXECUTE 'DROP TEXT SEARCH DICTIONARY IF EXISTS ' || lang || '_nostop CASCADE;
+CREATE TEXT SEARCH DICTIONARY ' || lang || '_nostop (TEMPLATE=pg_catalog.snowball, language=''' || lang || ''');
+COMMENT ON TEXT SEARCH DICTIONARY ' || lang || '_nostop IS ''' ||lang || ' snowball stemmer with no stopwords for ASCII words only.'';
+CREATE TEXT SEARCH CONFIGURATION ' || lang || '_nostop ( COPY = pg_catalog.' || lang || ' );
+ALTER TEXT SEARCH CONFIGURATION ' || lang || '_nostop ALTER MAPPING FOR word, hword, hword_part WITH pg_catalog.simple;
+ALTER TEXT SEARCH CONFIGURATION ' || lang || '_nostop ALTER MAPPING FOR asciiword, asciihword, hword_asciipart WITH ' || lang || '_nostop;';
+
+END LOOP;
+END;
+$$;
+--CREATE TEXT SEARCH CONFIGURATION title ( COPY = english_nostop );
+--CREATE TEXT SEARCH CONFIGURATION author ( COPY = english_nostop );
+--CREATE TEXT SEARCH CONFIGURATION subject ( COPY = english_nostop );
+CREATE TEXT SEARCH CONFIGURATION keyword ( COPY = english_nostop );
+--CREATE TEXT SEARCH CONFIGURATION identifier ( COPY = english_nostop );
+--CREATE TEXT SEARCH CONFIGURATION series ( COPY = english_nostop );
+CREATE TEXT SEARCH CONFIGURATION "default" ( COPY = english_nostop );
+
+
+COMMIT;
diff --git a/Open-ILS/src/sql/Pg/000.english.pg92.fts-config.sql b/Open-ILS/src/sql/Pg/000.english.pg92.fts-config.sql
index fd3fe58..0b24fd9 120000
--- a/Open-ILS/src/sql/Pg/000.english.pg92.fts-config.sql
+++ b/Open-ILS/src/sql/Pg/000.english.pg92.fts-config.sql
@@ -1 +1 @@
-000.english.pg90.fts-config.sql
\ No newline at end of file
+000.english.pg91.fts-config.sql
\ No newline at end of file
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index eb7b60e..11dd989 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -169,7 +169,11 @@ CREATE TABLE config.metabib_class (
name TEXT PRIMARY KEY,
label TEXT NOT NULL UNIQUE,
buoyant BOOL DEFAULT FALSE NOT NULL,
- restrict BOOL DEFAULT FALSE NOT NULL
+ restrict BOOL DEFAULT FALSE NOT NULL,
+ a_weight NUMERIC DEFAULT 1.0 NOT NULL,
+ b_weight NUMERIC DEFAULT 0.4 NOT NULL,
+ c_weight NUMERIC DEFAULT 0.2 NOT NULL,
+ d_weight NUMERIC DEFAULT 0.1 NOT NULL
);
CREATE TABLE config.metabib_field (
@@ -198,6 +202,49 @@ $$;
CREATE UNIQUE INDEX config_metabib_field_class_name_idx ON config.metabib_field (field_class, name);
+CREATE TABLE config.ts_config_list (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL
+);
+COMMENT ON TABLE config.ts_config_list IS $$
+Full Text Configs
+
+A list of full text configs with names and descriptions.
+$$;
+
+CREATE TABLE config.metabib_class_ts_map (
+ id SERIAL PRIMARY KEY,
+ field_class TEXT NOT NULL REFERENCES config.metabib_class (name),
+ ts_config TEXT NOT NULL REFERENCES config.ts_config_list (id),
+ active BOOL NOT NULL DEFAULT TRUE,
+ index_weight CHAR(1) NOT NULL DEFAULT 'C' CHECK (index_weight IN ('A','B','C','D')),
+ index_lang TEXT NULL,
+ search_lang TEXT NULL,
+ always BOOL NOT NULL DEFAULT true
+);
+COMMENT ON TABLE config.metabib_class_ts_map IS $$
+Text Search Configs for metabib class indexing
+
+This table contains text search config definitions for
+storing index_vector values.
+$$;
+
+CREATE TABLE config.metabib_field_ts_map (
+ id SERIAL PRIMARY KEY,
+ metabib_field INT NOT NULL REFERENCES config.metabib_field (id),
+ ts_config TEXT NOT NULL REFERENCES config.ts_config_list (id),
+ active BOOL NOT NULL DEFAULT TRUE,
+ index_weight CHAR(1) NOT NULL DEFAULT 'C' CHECK (index_weight IN ('A','B','C','D')),
+ index_lang TEXT NULL,
+ search_lang TEXT NULL
+);
+COMMENT ON TABLE config.metabib_field_ts_map IS $$
+Text Search Configs for metabib field indexing
+
+This table contains text search config definitions for
+storing index_vector values.
+$$;
+
CREATE TABLE config.metabib_search_alias (
alias TEXT PRIMARY KEY,
field_class TEXT NOT NULL REFERENCES config.metabib_class (name),
@@ -784,75 +831,6 @@ BEGIN
END;
$f$ LANGUAGE PLPGSQL;
-CREATE OR REPLACE FUNCTION oils_tsearch2 () RETURNS TRIGGER AS $$
-DECLARE
- normalizer RECORD;
- value TEXT := '';
-BEGIN
-
- value := NEW.value;
-
- IF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
- FOR normalizer IN
- SELECT n.func AS func,
- n.param_count AS param_count,
- m.params AS params
- FROM config.index_normalizer n
- JOIN config.metabib_field_index_norm_map m ON (m.norm = n.id)
- WHERE field = NEW.field AND m.pos < 0
- ORDER BY m.pos LOOP
- EXECUTE 'SELECT ' || normalizer.func || '(' ||
- quote_literal( value ) ||
- CASE
- WHEN normalizer.param_count > 0
- THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
- ELSE ''
- END ||
- ')' INTO value;
-
- END LOOP;
-
- NEW.value := value;
- END IF;
-
- IF NEW.index_vector = ''::tsvector THEN
- RETURN NEW;
- END IF;
-
- IF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
- FOR normalizer IN
- SELECT n.func AS func,
- n.param_count AS param_count,
- m.params AS params
- FROM config.index_normalizer n
- JOIN config.metabib_field_index_norm_map m ON (m.norm = n.id)
- WHERE field = NEW.field AND m.pos >= 0
- ORDER BY m.pos LOOP
- EXECUTE 'SELECT ' || normalizer.func || '(' ||
- quote_literal( value ) ||
- CASE
- WHEN normalizer.param_count > 0
- THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
- ELSE ''
- END ||
- ')' INTO value;
-
- END LOOP;
- END IF;
-
- IF TG_TABLE_NAME::TEXT ~ 'browse_entry$' THEN
- value := ARRAY_TO_STRING(
- evergreen.regexp_split_to_array(value, E'\\W+'), ' '
- );
- value := public.search_normalize(value);
- END IF;
-
- NEW.index_vector = to_tsvector((TG_ARGV[0])::regconfig, value);
-
- RETURN NEW;
-END;
-$$ LANGUAGE PLPGSQL;
-
-- List applied db patches that are deprecated by (and block the application of) my_db_patch
CREATE OR REPLACE FUNCTION evergreen.upgrade_list_applied_deprecates ( my_db_patch TEXT ) RETURNS SETOF evergreen.patch AS $$
SELECT DISTINCT l.version
diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index e833ef2..da9d4dd 100644
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -44,6 +44,14 @@ CREATE INDEX metabib_identifier_field_entry_index_vector_idx ON metabib.identifi
CREATE INDEX metabib_identifier_field_entry_value_idx ON metabib.identifier_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_identifier_field_entry_source_idx ON metabib.identifier_field_entry (source);
+CREATE TABLE metabib.combined_identifier_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_identifier_field_entry_fakepk_idx ON metabib.combined_identifier_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_identifier_field_entry_index_vector_idx ON metabib.combined_identifier_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_identifier_field_source_idx ON metabib.combined_identifier_field_entry (metabib_field);
CREATE TABLE metabib.title_field_entry (
id BIGSERIAL PRIMARY KEY,
@@ -60,6 +68,14 @@ CREATE INDEX metabib_title_field_entry_index_vector_idx ON metabib.title_field_e
CREATE INDEX metabib_title_field_entry_value_idx ON metabib.title_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_title_field_entry_source_idx ON metabib.title_field_entry (source);
+CREATE TABLE metabib.combined_title_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_title_field_entry_fakepk_idx ON metabib.combined_title_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_title_field_entry_index_vector_idx ON metabib.combined_title_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_title_field_source_idx ON metabib.combined_title_field_entry (metabib_field);
CREATE TABLE metabib.author_field_entry (
id BIGSERIAL PRIMARY KEY,
@@ -76,6 +92,14 @@ CREATE INDEX metabib_author_field_entry_index_vector_idx ON metabib.author_field
CREATE INDEX metabib_author_field_entry_value_idx ON metabib.author_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_author_field_entry_source_idx ON metabib.author_field_entry (source);
+CREATE TABLE metabib.combined_author_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_author_field_entry_fakepk_idx ON metabib.combined_author_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_author_field_entry_index_vector_idx ON metabib.combined_author_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_author_field_source_idx ON metabib.combined_author_field_entry (metabib_field);
CREATE TABLE metabib.subject_field_entry (
id BIGSERIAL PRIMARY KEY,
@@ -92,6 +116,14 @@ CREATE INDEX metabib_subject_field_entry_index_vector_idx ON metabib.subject_fie
CREATE INDEX metabib_subject_field_entry_value_idx ON metabib.subject_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_subject_field_entry_source_idx ON metabib.subject_field_entry (source);
+CREATE TABLE metabib.combined_subject_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_subject_field_entry_fakepk_idx ON metabib.combined_subject_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_subject_field_entry_index_vector_idx ON metabib.combined_subject_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_subject_field_source_idx ON metabib.combined_subject_field_entry (metabib_field);
CREATE TABLE metabib.keyword_field_entry (
id BIGSERIAL PRIMARY KEY,
@@ -108,6 +140,14 @@ CREATE INDEX metabib_keyword_field_entry_index_vector_idx ON metabib.keyword_fie
CREATE INDEX metabib_keyword_field_entry_value_idx ON metabib.keyword_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_keyword_field_entry_source_idx ON metabib.keyword_field_entry (source);
+CREATE TABLE metabib.combined_keyword_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_keyword_field_entry_fakepk_idx ON metabib.combined_keyword_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_keyword_field_entry_index_vector_idx ON metabib.combined_keyword_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_keyword_field_source_idx ON metabib.combined_keyword_field_entry (metabib_field);
CREATE TABLE metabib.series_field_entry (
id BIGSERIAL PRIMARY KEY,
@@ -124,6 +164,14 @@ CREATE INDEX metabib_series_field_entry_index_vector_idx ON metabib.series_field
CREATE INDEX metabib_series_field_entry_value_idx ON metabib.series_field_entry (SUBSTRING(value,1,1024)) WHERE index_vector = ''::TSVECTOR;
CREATE INDEX metabib_series_field_entry_source_idx ON metabib.series_field_entry (source);
+CREATE TABLE metabib.combined_series_field_entry (
+ record BIGINT NOT NULL,
+ metabib_field INT NULL,
+ index_vector tsvector NOT NULL
+);
+CREATE UNIQUE INDEX metabib_combined_series_field_entry_fakepk_idx ON metabib.combined_series_field_entry (record, COALESCE(metabib_field::TEXT,''));
+CREATE INDEX metabib_combined_series_field_entry_index_vector_idx ON metabib.combined_series_field_entry USING GIST (index_vector);
+CREATE INDEX metabib_combined_series_field_source_idx ON metabib.combined_series_field_entry (metabib_field);
CREATE TABLE metabib.facet_entry (
id BIGSERIAL PRIMARY KEY,
@@ -473,6 +521,59 @@ END;
$func$ LANGUAGE PLPGSQL;
+CREATE OR REPLACE FUNCTION metabib.update_combined_index_vectors(bib_id BIGINT) RETURNS VOID AS $func$
+BEGIN
+ DELETE FROM metabib.combined_keyword_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_keyword_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.keyword_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_keyword_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.keyword_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_title_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_title_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.title_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_title_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.title_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_author_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_author_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.author_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_author_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.author_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_subject_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_subject_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.subject_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_subject_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.subject_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_series_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_series_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.series_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_series_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.series_field_entry WHERE source = bib_id;
+
+ DELETE FROM metabib.combined_identifier_field_entry WHERE record = bib_id;
+ INSERT INTO metabib.combined_identifier_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, field, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.identifier_field_entry WHERE source = bib_id GROUP BY field;
+ INSERT INTO metabib.combined_identifier_field_entry(record, metabib_field, index_vector)
+ SELECT bib_id, NULL, strip(COALESCE(string_agg(index_vector::TEXT,' '),'')::tsvector)
+ FROM metabib.identifier_field_entry WHERE source = bib_id;
+
+END;
+$func$ LANGUAGE PLPGSQL;
+
CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$
DECLARE
fclass RECORD;
@@ -538,6 +639,10 @@ BEGIN
END LOOP;
+ IF NOT skip_search THEN
+ PERFORM metabib.update_combined_index_vectors(bib_id);
+ END IF;
+
RETURN;
END;
$func$ LANGUAGE PLPGSQL;
@@ -1491,4 +1596,74 @@ SELECT DISTINCT
END;
$func$ LANGUAGE PLPGSQL;
+CREATE OR REPLACE FUNCTION public.oils_tsearch2 () RETURNS TRIGGER AS $$
+DECLARE
+ normalizer RECORD;
+ value TEXT := '';
+ temp_vector TEXT := '';
+ ts_rec RECORD;
+ cur_weight "char";
+BEGIN
+
+ value := NEW.value;
+ NEW.index_vector = ''::tsvector;
+
+ IF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
+ FOR normalizer IN
+ SELECT n.func AS func,
+ n.param_count AS param_count,
+ m.params AS params
+ FROM config.index_normalizer n
+ JOIN config.metabib_field_index_norm_map m ON (m.norm = n.id)
+ WHERE field = NEW.field
+ ORDER BY m.pos LOOP
+ EXECUTE 'SELECT ' || normalizer.func || '(' ||
+ quote_literal( value ) ||
+ CASE
+ WHEN normalizer.param_count > 0
+ THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
+ ELSE ''
+ END ||
+ ')' INTO value;
+
+ END LOOP;
+ NEW.value = value;
+ END IF;
+
+ IF TG_TABLE_NAME::TEXT ~ 'browse_entry$' THEN
+ value := ARRAY_TO_STRING(
+ evergreen.regexp_split_to_array(value, E'\\W+'), ' '
+ );
+ value := public.search_normalize(value);
+ NEW.index_vector = to_tsvector(TG_ARGV[0]::regconfig, value);
+ ELSIF TG_TABLE_NAME::TEXT ~ 'field_entry$' THEN
+ FOR ts_rec IN
+ SELECT ts_config, index_weight
+ FROM config.metabib_class_ts_map
+ WHERE field_class = TG_ARGV[0]
+ AND index_lang IS NULL OR EXISTS (SELECT 1 FROM metabib.record_attr WHERE id = NEW.source AND index_lang IN(attrs->'item_lang',attrs->'language'))
+ AND always OR NOT EXISTS (SELECT 1 FROM config.metabib_field_ts_map WHERE metabib_field = NEW.field)
+ UNION
+ SELECT ts_config, index_weight
+ FROM config.metabib_field_ts_map
+ WHERE metabib_field = NEW.field
+ AND index_lang IS NULL OR EXISTS (SELECT 1 FROM metabib.record_attr WHERE id = NEW.source AND index_lang IN(attrs->'item_lang',attrs->'language'))
+ ORDER BY index_weight ASC
+ LOOP
+ IF cur_weight IS NOT NULL AND cur_weight != ts_rec.index_weight THEN
+ NEW.index_vector = NEW.index_vector || setweight(temp_vector::tsvector,cur_weight);
+ temp_vector = '';
+ END IF;
+ cur_weight = ts_rec.index_weight;
+ SELECT INTO temp_vector temp_vector || ' ' || to_tsvector(ts_rec.ts_config::regconfig, value)::TEXT;
+ END LOOP;
+ NEW.index_vector = NEW.index_vector || setweight(temp_vector::tsvector,cur_weight);
+ ELSE
+ NEW.index_vector = to_tsvector(TG_ARGV[0]::regconfig, value);
+ END IF;
+
+ RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
COMMIT;
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index 8ee95df..9beb73e 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -12362,3 +12362,35 @@ VALUES (
'bool',
NULL
);
+
+
+INSERT INTO config.ts_config_list(id, name) VALUES
+ ('simple','Non-Stemmed Simple'),
+ ('danish_nostop','Danish Stemmed'),
+ ('dutch_nostop','Dutch Stemmed'),
+ ('english_nostop','English Stemmed'),
+ ('finnish_nostop','Finnish Stemmed'),
+ ('french_nostop','French Stemmed'),
+ ('german_nostop','German Stemmed'),
+ ('hungarian_nostop','Hungarian Stemmed'),
+ ('italian_nostop','Italian Stemmed'),
+ ('norwegian_nostop','Norwegian Stemmed'),
+ ('portuguese_nostop','Portuguese Stemmed'),
+ ('romanian_nostop','Romanian Stemmed'),
+ ('russian_nostop','Russian Stemmed'),
+ ('spanish_nostop','Spanish Stemmed'),
+ ('swedish_nostop','Swedish Stemmed'),
+ ('turkish_nostop','Turkish Stemmed');
+
+INSERT INTO config.metabib_class_ts_map(field_class, ts_config, index_weight, always) VALUES
+ ('keyword','simple','A',true),
+ ('keyword','english_nostop','C',true),
+ ('title','simple','A',true),
+ ('title','english_nostop','C',true),
+ ('author','simple','A',true),
+ ('author','english_nostop','C',true),
+ ('series','simple','A',true),
+ ('series','english_nostop','C',true),
+ ('subject','simple','A',true),
+ ('subject','english_nostop','C',true),
+ ('identifier','simple','A',true);
diff --git a/Open-ILS/src/sql/Pg/999.functions.global.sql b/Open-ILS/src/sql/Pg/999.functions.global.sql
index 02f2861..d6ed10b 100644
--- a/Open-ILS/src/sql/Pg/999.functions.global.sql
+++ b/Open-ILS/src/sql/Pg/999.functions.global.sql
@@ -2165,4 +2165,25 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION evergreen.rel_bump(terms TEXT[], value TEXT, bumps TEXT[], mults NUMERIC[]) RETURNS NUMERIC AS
+$BODY$
+use strict;
+my ($terms,$value,$bumps,$mults) = @_;
+
+my $retval = 1;
+
+for (my $id = 0; $id < @$bumps; $id++) {
+ if ($bumps->[$id] eq 'first_word') {
+ $retval *= $mults->[$id] if ($value =~ /^$terms->[0]/);
+ } elsif ($bumps->[$id] eq 'full_match') {
+ my $fullmatch = join(' ', @$terms);
+ $retval *= $mults->[$id] if ($value =~ /^$fullmatch$/);
+ } elsif ($bumps->[$id] eq 'word_order') {
+ my $wordorder = join('.*', @$terms);
+ $retval *= $mults->[$id] if ($value =~ /$wordorder/);
+ }
+}
+return $retval;
+$BODY$ LANGUAGE plperlu IMMUTABLE STRICT COST 100;
+
-- user activity functions --
commit 3249c78b060fb6bcf00964b825c3a77c332f73a4
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Wed Oct 10 17:23:43 2012 -0400
QueryParser Driver: Remove Unphrases, add negates
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 16f4731..9afb754 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -929,18 +929,15 @@ sub flatten {
}
}
+ my $NOT = '';
+ $NOT = 'NOT ' if $node->negate;
- $where .= '(' . $talias . ".id IS NOT NULL";
+ $where .= "$NOT(" . $talias . ".id IS NOT NULL";
if (@{$node->phrases}) {
$where .= ' AND ' . join(' AND ', map {
"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)
} @{$node->phrases});
}
- if (@{$node->unphrases}) {
- $where .= ' AND ' . join(' AND ', map {
- "${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)
- } @{$node->unphrases});
- }
for my $atom (@{$node->only_real_atoms}) {
next unless $atom->{content} && $atom->{content} =~ /(^\^|\$$)/;
$where .= " AND ${talias}.value ~* ".$self->QueryParser->quote_phrase_value($atom->{content});
@@ -989,8 +986,11 @@ sub flatten {
push(@rank_list, @{$$subnode{rank_list}});
$from .= $$subnode{from};
+ my $NOT = '';
+ $NOT = 'NOT ' if $node->negate;
+
if ($$subnode{where} ne '') {
- $where .= "(\n"
+ $where .= "$NOT(\n"
. ${spc} x ($self->plan_level + 6) . $$subnode{where} . "\n"
. ${spc} x ($self->plan_level + 5) . ')';
}
commit 6a90104750160b087a0252d6775ce5eb61d533d5
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Wed Oct 10 16:51:36 2012 -0400
Use ][ instead of # to split facets
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 02deeeb..1aa5c76 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -919,7 +919,7 @@ sub decompose {
my $phrase_cleanup_re = qr/\s*(\Q$required_op\E|\Q$disallowed_op\E|\Q$and_op\E|\Q$or_op\E|\Q$group_start\E|\Q$group_end\E|\Q$float_start\E|\Q$float_end\E|\Q$modifier_tag\E|\Q$negated_op\E|:|\(|\))/;
# Build the filter and modifier uber-regexps
- my $facet_re = '^\s*(-?)((?:' . join( '|', @{$pkg->facet_classes}) . ')(?:\|\w+)*)\[(.+?)\]';
+ my $facet_re = '^\s*(-?)((?:' . join( '|', @{$pkg->facet_classes}) . ')(?:\|\w+)*)\[(.+?)\](?!\[)';
warn ' 'x$recursing." ** Facet RE: $facet_re\n" if $self->debug;
my $filter_re = '^\s*(-?)(' . join( '|', @{$pkg->filters}) . ')\(([^()]+)\)';
@@ -1123,7 +1123,7 @@ sub decompose {
my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
my $facet = $2;
- my $facet_value = [ split '\s*#\s*', $3 ];
+ my $facet_value = [ split '\s*\]\[\s*', $3 ];
$struct->new_facet( $facet => $facet_value, $negate );
$_ = $';
commit bdbec2aadf744331c25c27983402acbdfbe8396a
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Thu Sep 27 16:48:03 2012 -0400
QueryParser Driver: Add "lucky" modifier
Forces to 1 result. Best used with redirect on single hit active.
Because why not.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 9fff3de..16f4731 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -520,6 +520,7 @@ __PACKAGE__->add_search_filter( 'superpage_size' );
__PACKAGE__->add_search_filter( 'estimation_strategy' );
__PACKAGE__->add_search_modifier( 'available' );
__PACKAGE__->add_search_modifier( 'staff' );
+__PACKAGE__->add_search_modifier( 'lucky' );
# Start from container data (bre, acn, acp): container(bre,bookbag,123,deadb33fdeadb33fdeadb33fdeadb33f)
__PACKAGE__->add_search_filter( 'container' );
@@ -626,6 +627,7 @@ sub toSQL {
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->modifiers});
my $core_limit = $self->QueryParser->core_limit || 25000;
+ $core_limit = 1 if($self->find_modifier('lucky'));
my $flat_where = $$flat_plan{where};
if ($flat_where ne '') {
commit 0662b43a449f33fbb659b94a222da099d342a19c
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 18 11:55:07 2012 -0400
QueryParser Driver: Improve format filter
Allow multi-select in particular, and make negate more intuitive.
-format(at-d) would previously generate:
-item_type(a,t) -item_form(d)
Now it generates:
-(item_type(a,t) item_form(d))
Multi-select allows for things like:
format(at-d,g)
To generate:
((item_type(a,t) item_form(d)) || item_type(g))
Negating that results in:
-((item_type(a,t) item_form(d)) || item_type(g))
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index bb12b75..9fff3de 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -67,13 +67,19 @@ sub format_callback {
my $return = '';
my $negate_flag = ($negate ? '-' : '');
- if(@$params[0]) {
- my ($t,$f) = split('-', @$params[0]);
- $return .= $negate_flag .'item_type(' . join(',',split('', $t)) . ')' if ($t);
- $return .= ' ' if ($t and $f);
- $return .= $negate_flag .'item_form(' . join(',',split('', $f)) . ')' if ($f);
- $return = '(' . $return . ')' if ($t and $f);
+ my @returns;
+ for my $param (@$params) {
+ my ($t,$f) = split('-', $param);
+ my $treturn = '';
+ $treturn .= 'item_type(' . join(',',split('', $t)) . ')' if ($t);
+ $treturn .= ' ' if ($t and $f);
+ $treturn .= 'item_form(' . join(',',split('', $f)) . ')' if ($f);
+ $treturn = '(' . $treturn . ')' if ($t and $f);
+ push(@returns, $treturn) if $treturn;
}
+ $return = join(' || ', @returns);
+ $return = '(' . $return . ')' if(@returns > 1);
+ $return = $negate_flag.$return if($return);
return $return;
}
commit b4fb02f964b88d4848a2fb0e2d242ed8b3cb5fcf
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Fri Sep 14 12:15:40 2012 -0400
QueryParser Driver: Long Line Cleanup
Both in the code and in the generated where clause.
The where clause we start a new line whenever:
1 - We encounter an AND or OR
2 - We are building a complex subquery (including embedded newlines)
3 - We enter a subplan
This makes for a nicely human-readable where clause.
For the code we split many long lines into multiple. A number of those were
changed due to the where clause formatting.
We also change all instances of multiple ${spc} being added to use the
${spc} x #
method of doing things, as it tends to be shorter.
Also, we move some conditionals from the ends of lines to the fronts, mainly
in those situations where we are moving something from single to multi line.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index a962998..bb12b75 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -574,7 +574,13 @@ sub toSQL {
# generate the relevance ranking
my $rel = '1'; # Default to something simple in case rank_list is empty.
- $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1" if (@{$$flat_plan{rank_list}});
+ if (@{$$flat_plan{rank_list}}) {
+ $rel = "AVG(\n"
+ . ${spc} x 5 ."("
+ . join(")\n" . ${spc} x 5 . "+ (", @{$$flat_plan{rank_list}})
+ . ")\n"
+ . ${spc} x 4 . ")+1";
+ }
# find any supplied sort option
my ($sort_filter) = $self->find_filter('sort');
@@ -616,10 +622,8 @@ sub toSQL {
my $core_limit = $self->QueryParser->core_limit || 25000;
my $flat_where = $$flat_plan{where};
- if ($flat_where eq '()') {
- $flat_where = '';
- } else {
- $flat_where = "AND $flat_where";
+ if ($flat_where ne '') {
+ $flat_where = "AND (\n" . ${spc} x 5 . $flat_where . "\n" . ${spc} x 4 . ")";
}
my $site = $self->find_filter('site');
@@ -846,7 +850,7 @@ sub flatten {
my $self = shift;
my $from = shift || '';
- my $where = shift || '(';
+ my $where = shift || '';
my $with = '';
my @rank_list;
@@ -866,16 +870,17 @@ sub flatten {
my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 0.0)";
- my $core_limit = $self->QueryParser->core_limit || 25000;
- $from .= "\n${spc}${spc}${spc}${spc}LEFT JOIN (\n${spc}${spc}${spc}${spc}${spc}SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq /* search */\n${spc}${spc}${spc}${spc}${spc} FROM $table AS fe";
- $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
+ $from .= "\n" . ${spc} x 4 ."LEFT JOIN (\n"
+ . ${spc} x 5 . "SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq /* search */\n"
+ . ${spc} x 6 . "FROM $table AS fe";
+ $from .= "\n" . ${spc} x 7 . "JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
if ($node->dummy_count < @{$node->only_atoms} ) {
$with .= ",\n " if $with;
$with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq )";
- $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
+ $from .= "\n" . ${spc} x 6 . "JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
} else {
- $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}, (SELECT NULL::tsquery AS tsq ) AS ${talias}_xq";
+ $from .= "\n" . ${spc} x 6 . ", (SELECT NULL::tsquery AS tsq ) AS ${talias}_xq";
}
my @bump_fields;
@@ -890,7 +895,7 @@ sub flatten {
} @bump_fields
);
if (@field_ids) {
- $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}WHERE fe_weight.id IN (" .
+ $from .= "\n" . ${spc} x 6 . "WHERE fe_weight.id IN (" .
join(',', @field_ids) . ")";
}
@@ -898,8 +903,7 @@ sub flatten {
@bump_fields = @{$self->QueryParser->search_fields->{$node->classname}};
}
- ###$from .= "\n${spc}${spc}LIMIT $core_limit";
- $from .= "\n${spc}${spc}${spc}${spc}) AS $talias ON (m.source = ${talias}.source)";
+ $from .= "\n" . ${spc} x 4 . ") AS $talias ON (m.source = ${talias}.source)";
my %used_bumps;
@@ -913,14 +917,22 @@ sub flatten {
next if ($$bumps{$b}{multiplier} == 1); # optimization to remove unneeded bumps
my $bump_case = $self->rel_bump( $node, $b, $$bumps{$b}{multiplier} );
- $node_rank .= "\n${spc}${spc}${spc}${spc}${spc}* " . $bump_case if ($bump_case);
+ $node_rank .= "\n" . ${spc} x 5 . "* " . $bump_case if ($bump_case);
}
}
$where .= '(' . $talias . ".id IS NOT NULL";
- $where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
- $where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
+ if (@{$node->phrases}) {
+ $where .= ' AND ' . join(' AND ', map {
+ "${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)
+ } @{$node->phrases});
+ }
+ if (@{$node->unphrases}) {
+ $where .= ' AND ' . join(' AND ', map {
+ "${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)
+ } @{$node->unphrases});
+ }
for my $atom (@{$node->only_real_atoms}) {
next unless $atom->{content} && $atom->{content} =~ /(^\^|\$$)/;
$where .= " AND ${talias}.value ~* ".$self->QueryParser->quote_phrase_value($atom->{content});
@@ -942,16 +954,19 @@ sub flatten {
}
my $join_type = ($node->negate or !$self->top_plan) ? 'LEFT' : 'INNER';
- $from .= "\n${spc}$join_type JOIN /* facet */ metabib.facet_entry $talias ON (\n${spc}${spc}m.source = ${talias}.source\n${spc}${spc}".
- "AND SUBSTRING(${talias}.value,1,1024) IN (" . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n${spc}${spc}".
- "AND ${talias}.field IN (". join(',', @field_ids) . ")\n${spc})";
+ $from .= "\n${spc}$join_type JOIN /* facet */ metabib.facet_entry $talias ON (\n"
+ . ${spc} x 2 . "m.source = ${talias}.source\n"
+ . ${spc} x 2 . "AND SUBSTRING(${talias}.value,1,1024) IN ("
+ . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n"
+ . ${spc} x 2 ."AND ${talias}.field IN (". join(',', @field_ids) . ")\n"
+ . "${spc})";
if ($join_type ne 'INNER') {
my $NOT = $node->negate ? '' : ' NOT';
$where .= "${talias}.id IS$NOT NULL";
- } elsif ($where ne '(') {
+ } elsif ($where ne '') {
# Strip extra joiner
- $where =~ s/\s(AND|OR)\s$//;
+ $where =~ s/(\s|\n)+(AND|OR)\s$//;
}
} else {
@@ -959,14 +974,18 @@ sub flatten {
# strip the trailing bool from the previous loop if there is
# nothing to add to the where within this loop.
- if ($$subnode{where} eq '()') {
- $where =~ s/\s(AND|OR)\s$//;
+ if ($$subnode{where} eq '') {
+ $where =~ s/(\s|\n)+(AND|OR)\s$//;
}
push(@rank_list, @{$$subnode{rank_list}});
$from .= $$subnode{from};
- $where .= "$$subnode{where}" unless $$subnode{where} eq '()';
+ if ($$subnode{where} ne '') {
+ $where .= "(\n"
+ . ${spc} x ($self->plan_level + 6) . $$subnode{where} . "\n"
+ . ${spc} x ($self->plan_level + 5) . ')';
+ }
if ($$subnode{with}) {
$with .= ",\n " if $with;
@@ -977,14 +996,14 @@ sub flatten {
warn "flatten(): appending WHERE bool to: $where\n" if $self->QueryParser->debug;
- if ($where ne '(') {
- $where .= ' AND ' if ($node eq '&');
- $where .= ' OR ' if ($node eq '|');
+ if ($where ne '') {
+ $where .= "\n" . ${spc} x ( $self->plan_level + 5 ) . 'AND ' if ($node eq '&');
+ $where .= "\n" . ${spc} x ( $self->plan_level + 5 ) . 'OR ' if ($node eq '|');
}
}
}
- my $joiner = sprintf(" %s ", ($self->joiner eq '&' ? 'AND' : 'OR'));
+ my $joiner = "\n" . ${spc} x ( $self->plan_level + 5 ) . ($self->joiner eq '&' ? 'AND ' : 'OR ');
# for each dynamic filter, build more of the WHERE clause
for my $filter (@{$self->filters}) {
my $NOT = $filter->negate ? 'NOT ' : '';
@@ -994,7 +1013,7 @@ sub flatten {
if $self->QueryParser->debug;
# bool joiner for intra-plan nodes/filters
- $where .= $joiner if $where ne '(';
+ $where .= $joiner if $where ne '';
my @fargs = @{$filter->args};
my $fname = $filter->name;
@@ -1010,23 +1029,33 @@ sub flatten {
} else {
if ($filter->name eq 'before') {
if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
+ $where .= $joiner if $where ne '';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') <= "
+ . $self->QueryParser->quote_value($filter->args->[0])
+ . ", false)";
}
} elsif ($filter->name eq 'after') {
if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
+ $where .= $joiner if $where ne '';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') >= "
+ . $self->QueryParser->quote_value($filter->args->[0])
+ . ", false)";
}
} elsif ($filter->name eq 'during') {
if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE(" . $self->QueryParser->quote_value($filter->args->[0]) . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2'), false)";
+ $where .= $joiner if $where ne '';
+ $where .= "${NOT}COALESCE("
+ . $self->QueryParser->quote_value($filter->args->[0])
+ . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2'), false)";
}
} elsif ($filter->name eq 'between') {
if (@{$filter->args} == 2) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]) . ", false)";
+ $where .= $joiner if $where ne '';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') BETWEEN "
+ . $self->QueryParser->quote_value($filter->args->[0])
+ . " AND "
+ . $self->QueryParser->quote_value($filter->args->[1])
+ . ", false)";
}
} elsif ($filter->name eq 'container') {
if (@{$filter->args} >= 3) {
@@ -1051,47 +1080,112 @@ sub flatten {
if ($class) {
my ($u,$e) = $apputils->checksesperm($token) if ($token);
$perm_join = ' OR c.owner = ' . $u->id if ($u && !$e);
- $where .= $joiner if $where ne '(';
- $where .= '(' if $class eq 'copy';
- $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)";
- }
- if ($class eq 'copy') {
- my $subjoiner = $filter->negate ? ' AND ' : ' OR ';
- $where .= "$subjoiner${NOT}EXISTS(SELECT 1 FROM container.copy_bucket_item ci JOIN container.copy_bucket c ON (c.id = ci.bucket) JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy WHERE c.btype = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND pr.peer_record = m.source LIMIT 1))";
+ $where .= $joiner if $where ne '';
+ my $spcdepth = $self->plan_level + 5;
+ if($class eq 'copy') {
+ $spcdepth += 1;
+ $where .= "(\n" . ${spc} x $spcdepth;
+ }
+ $where .= "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 1) . "SELECT 1 FROM container.${class}_bucket_item ci\n"
+ . ${spc} x ($spcdepth + 4) . "JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join\n"
+ . ${spc} x ($spcdepth + 1) . "WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . "\n"
+ . ${spc} x ($spcdepth + 4) . "AND c.id = " . $self->QueryParser->quote_value($cid) . "\n"
+ . ${spc} x ($spcdepth + 4) . "AND (c.pub IS TRUE$perm_join)\n"
+ . ${spc} x ($spcdepth + 4) . "AND $rec_field = m.source\n"
+ . ${spc} x ($spcdepth + 1) . "LIMIT 1\n"
+ . ${spc} x $spcdepth . ")";
+ if ($class eq 'copy') {
+ my $subjoiner = $filter->negate ? 'AND' : 'OR';
+ $where .= "\n"
+ . ${spc} x ($spcdepth) . $subjoiner . "\n"
+ . ${spc} x ($spcdepth) . "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 1) . "SELECT 1 FROM container.copy_bucket_item ci\n"
+ . ${spc} x ($spcdepth + 4) . "JOIN container.copy_bucket c ON (c.id = ci.bucket)\n"
+ . ${spc} x ($spcdepth + 4) . "JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy\n"
+ . ${spc} x ($spcdepth + 1) . "WHERE c.btype = " . $self->QueryParser->quote_value($cid) . "\n"
+ . ${spc} x ($spcdepth + 4) . "AND (c.pub IS TRUE$perm_join)\n"
+ . ${spc} x ($spcdepth + 4) . "AND pr.peer_record = m.source\n"
+ . ${spc} x ($spcdepth + 1) . "LIMIT 1\n"
+ . ${spc} x $spcdepth . ")\n"
+ . ${spc} x ($spcdepth - 1) . ")";
+ }
}
}
} elsif ($filter->name eq 'record_list') {
if (@{$filter->args} > 0) {
my $key = 'm.source';
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers});
- $where .= $joiner if $where ne '(';
+ $where .= $joiner if $where ne '';
$where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{$filter->args}) . ')';
}
} elsif ($filter->name eq 'locations') {
if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
- $where .= $filter->negate ? ' AND ' : ' OR ';
- $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ my $spcdepth = $self->plan_level + 5;
+ $where .= $joiner if $where ne '';
+ $where .= "(\n"
+ . ${spc} x ($spcdepth + 1) . "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 2) . "SELECT 1 FROM asset.call_number acn\n"
+ . ${spc} x ($spcdepth + 5) . "JOIN asset.copy acp ON acn.id = acp.call_number\n"
+ . ${spc} x ($spcdepth + 2) . "WHERE m.source = acn.record\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.circ_lib IN (SELECT * FROM search_org_list)\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acn.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acp.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")\n"
+ . ${spc} x ($spcdepth + 2) . "LIMIT 1\n"
+ . ${spc} x ($spcdepth + 1) . ")\n"
+ . ${spc} x ($spcdepth + 1) . ($filter->negate ? 'AND' : 'OR') . "\n"
+ . ${spc} x ($spcdepth + 1) . "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 2) . "SELECT 1 FROM biblio.peer_bib_copy_map pr\n"
+ . ${spc} x ($spcdepth + 5) . "JOIN asset.copy acp ON pr.target_copy = acp.id\n"
+ . ${spc} x ($spcdepth + 2) . "WHERE m.source = pr.peer_record\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.circ_lib IN (SELECT * FROM search_org_list)\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acp.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")\n"
+ . ${spc} x ($spcdepth + 2) . "LIMIT 1\n"
+ . ${spc} x ($spcdepth + 1) . ")\n"
+ . ${spc} x $spcdepth . ")";
}
} elsif ($filter->name eq 'statuses') {
if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
- $where .= $filter->negate ? ' AND ' : ' OR ';
- $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ my $spcdepth = $self->plan_level + 5;
+ $where .= $joiner if $where ne '';
+ $where .= "(\n"
+ . ${spc} x ($spcdepth + 1) . "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 2) . "SELECT 1 FROM asset.call_number acn\n"
+ . ${spc} x ($spcdepth + 5) . "JOIN asset.copy acp ON acn.id = acp.call_number\n"
+ . ${spc} x ($spcdepth + 2) . "WHERE m.source = acn.record\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.circ_lib IN (SELECT * FROM search_org_list)\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acn.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acp.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")\n"
+ . ${spc} x ($spcdepth + 2) . "LIMIT 1\n"
+ . ${spc} x ($spcdepth + 1) . ")\n"
+ . ${spc} x ($spcdepth + 1) . ($filter->negate ? 'AND' : 'OR') . "\n"
+ . ${spc} x ($spcdepth + 1) . "${NOT}EXISTS(\n"
+ . ${spc} x ($spcdepth + 2) . "SELECT 1 FROM biblio.peer_bib_copy_map pr\n"
+ . ${spc} x ($spcdepth + 5) . "JOIN asset.copy acp ON pr.target_copy = acp.id\n"
+ . ${spc} x ($spcdepth + 2) . "WHERE m.source = pr.peer_record\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.circ_lib IN (SELECT * FROM search_org_list)\n"
+ . ${spc} x ($spcdepth + 5) . "AND NOT acp.deleted\n"
+ . ${spc} x ($spcdepth + 5) . "AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")\n"
+ . ${spc} x ($spcdepth + 2) . "LIMIT 1\n"
+ . ${spc} x ($spcdepth + 1) . ")\n"
+ . ${spc} x $spcdepth . ")";
}
} elsif ($filter->name eq 'bib_source') {
if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE(bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . "), false)";
+ $where .= $joiner if $where ne '';
+ $where .= "${NOT}COALESCE(bre.source IN ("
+ . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args })
+ . "), false)";
}
}
}
}
warn "flatten(): full filter where => $where\n" if $self->QueryParser->debug;
- return { rank_list => \@rank_list, from => $from, where => $where.')', with => $with };
+ return { rank_list => \@rank_list, from => $from, where => $where, with => $with };
}
@@ -1263,7 +1357,7 @@ sub tsquery {
for my $atom (@{$self->query_atoms}) {
if (ref($atom)) {
- $self->{tsquery} .= "\n${spc}${spc}${spc}" .$atom->sql;
+ $self->{tsquery} .= "\n" . ${spc} x 3 . $atom->sql;
} else {
$self->{tsquery} .= $atom x 2;
}
commit c05bd342e188daa04972d31c00909b81d056e78c
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Fri Sep 14 08:57:45 2012 -0400
QueryParser Driver: Remove Switch usage
Because not all distros install it by default.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 358972e..a962998 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -7,7 +7,6 @@ use base 'QueryParser';
use OpenSRF::Utils::JSON;
use OpenILS::Application::AppUtils;
use OpenILS::Utils::CStoreEditor;
-use Switch;
my $U = 'OpenILS::Application::AppUtils';
my ${spc} = ' ' x 2;
@@ -1009,98 +1008,83 @@ sub flatten {
warn "flatten(): filter where => $where\n"
if $self->QueryParser->debug;
} else {
- switch ($filter->name) {
- case 'before' {
- if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
- }
+ if ($filter->name eq 'before') {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
}
- case 'after' {
- if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
- }
+ } elsif ($filter->name eq 'after') {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
}
- case 'during' {
- if (@{$filter->args} == 1) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE(" . $self->QueryParser->quote_value($filter->args->[0]) . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2'), false)";
- }
+ } elsif ($filter->name eq 'during') {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}COALESCE(" . $self->QueryParser->quote_value($filter->args->[0]) . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2'), false)";
}
- case 'between' {
- if (@{$filter->args} == 2) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE((mrd.attrs->'date1') BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]) . ", false)";
- }
+ } elsif ($filter->name eq 'between') {
+ if (@{$filter->args} == 2) {
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]) . ", false)";
}
- case 'container' {
- if (@{$filter->args} >= 3) {
- my ($class, $ctype, $cid, $token) = @{$filter->args};
- my $perm_join = '';
- my $rec_join = '';
- my $rec_field = 'ci.target_biblio_record_entry';
- switch($class) {
- case 'bre' {
- $class = 'biblio_record_entry';
- }
- case 'acn' {
- $class = 'call_number';
- $rec_field = 'cn.record';
- $rec_join = 'JOIN asset.call_number cn ON (ci.target_call_number = cn.id)';
- }
- case 'acp' {
- $class = 'copy';
- $rec_field = 'cn.record';
- $rec_join = 'JOIN asset.copy cp ON (ci.target_copy = cp.id) JOIN asset.call_number cn ON (cp.call_number = cn.id)';
- }
- else {
- $class = undef;
- }
- }
-
- if ($class) {
- my ($u,$e) = $apputils->checksesperm($token) if ($token);
- $perm_join = ' OR c.owner = ' . $u->id if ($u && !$e);
- $where .= $joiner if $where ne '(';
- $where .= '(' if $class eq 'copy';
- $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)";
- }
- if ($class eq 'copy') {
- my $subjoiner = $filter->negate ? ' AND ' : ' OR ';
- $where .= "$subjoiner${NOT}EXISTS(SELECT 1 FROM container.copy_bucket_item ci JOIN container.copy_bucket c ON (c.id = ci.bucket) JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy WHERE c.btype = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND pr.peer_record = m.source LIMIT 1))";
- }
+ } elsif ($filter->name eq 'container') {
+ if (@{$filter->args} >= 3) {
+ my ($class, $ctype, $cid, $token) = @{$filter->args};
+ my $perm_join = '';
+ my $rec_join = '';
+ my $rec_field = 'ci.target_biblio_record_entry';
+ if ($class eq 'bre') {
+ $class = 'biblio_record_entry';
+ } elsif ($class eq 'acn') {
+ $class = 'call_number';
+ $rec_field = 'cn.record';
+ $rec_join = 'JOIN asset.call_number cn ON (ci.target_call_number = cn.id)';
+ } elsif ($class eq 'acp') {
+ $class = 'copy';
+ $rec_field = 'cn.record';
+ $rec_join = 'JOIN asset.copy cp ON (ci.target_copy = cp.id) JOIN asset.call_number cn ON (cp.call_number = cn.id)';
+ } else {
+ $class = undef;
}
- }
- case 'record_list' {
- if (@{$filter->args} > 0) {
- my $key = 'm.source';
- $key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers});
+
+ if ($class) {
+ my ($u,$e) = $apputils->checksesperm($token) if ($token);
+ $perm_join = ' OR c.owner = ' . $u->id if ($u && !$e);
$where .= $joiner if $where ne '(';
- $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{$filter->args}) . ')';
+ $where .= '(' if $class eq 'copy';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)";
}
- }
- case 'locations' {
- if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
- $where .= $filter->negate ? ' AND ' : ' OR ';
- $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ if ($class eq 'copy') {
+ my $subjoiner = $filter->negate ? ' AND ' : ' OR ';
+ $where .= "$subjoiner${NOT}EXISTS(SELECT 1 FROM container.copy_bucket_item ci JOIN container.copy_bucket c ON (c.id = ci.bucket) JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy WHERE c.btype = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND pr.peer_record = m.source LIMIT 1))";
}
}
- case 'statuses' {
- if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
- $where .= $filter->negate ? ' AND ' : ' OR ';
- $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
- }
+ } elsif ($filter->name eq 'record_list') {
+ if (@{$filter->args} > 0) {
+ my $key = 'm.source';
+ $key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers});
+ $where .= $joiner if $where ne '(';
+ $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{$filter->args}) . ')';
}
- case 'bib_source' {
- if (@{$filter->args} > 0) {
- $where .= $joiner if $where ne '(';
- $where .= "${NOT}COALESCE(bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . "), false)";
- }
+ } elsif ($filter->name eq 'locations') {
+ if (@{$filter->args} > 0) {
+ $where .= $joiner if $where ne '(';
+ $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
+ $where .= $filter->negate ? ' AND ' : ' OR ';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ }
+ } elsif ($filter->name eq 'statuses') {
+ if (@{$filter->args} > 0) {
+ $where .= $joiner if $where ne '(';
+ $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
+ $where .= $filter->negate ? ' AND ' : ' OR ';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ }
+ } elsif ($filter->name eq 'bib_source') {
+ if (@{$filter->args} > 0) {
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}COALESCE(bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . "), false)";
}
}
}
commit 0db54b56c88e540a44f1af2dbec126dd3c6409ba
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Wed Sep 12 09:12:41 2012 -0400
QueryParser Driver: Improve anchored searches
By checking individual atoms for ^ and $ anchors we can get better results,
without needing to have people quote individual terms.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index c0e345b..358972e 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -827,7 +827,7 @@ sub rel_bump {
my $bump = shift;
my $multiplier = shift;
- my $only_atoms = $node->only_atoms;
+ my $only_atoms = $node->only_real_atoms;
return '' if (!@$only_atoms);
if ($bump eq 'first_word') {
@@ -922,6 +922,10 @@ sub flatten {
$where .= '(' . $talias . ".id IS NOT NULL";
$where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
$where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
+ for my $atom (@{$node->only_real_atoms}) {
+ next unless $atom->{content} && $atom->{content} =~ /(^\^|\$$)/;
+ $where .= " AND ${talias}.value ~* ".$self->QueryParser->quote_phrase_value($atom->{content});
+ }
$where .= ')';
push @rank_list, $node_rank;
@@ -1230,6 +1234,18 @@ sub only_atoms {
return \@only_atoms;
}
+sub only_real_atoms {
+ my $self = shift;
+
+ my $atoms = $self->query_atoms;
+ my @only_real_atoms;
+ for my $a (@$atoms) {
+ push(@only_real_atoms, $a) if (ref($a) && $a->isa('QueryParser::query_plan::node::atom') && !($a->{dummy}));
+ }
+
+ return \@only_real_atoms;
+}
+
sub dummy_count {
my $self = shift;
return $self->{dummy_count};
commit a4f5bc9dce8fb4065879db02559ef044d31ea888
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 11 15:02:59 2012 -0400
QueryParser Driver: Protect against NULLs
mrd.attrs->'value' can return NULL. If this happens:
Checking that the value is within a range or list will work fine.
NEGATING that will not.
This is because:
AND NULL returns NULL
AND NOT (NULL) also returns NULL
The solution? Adjust things so we can wrap all the offending checks in a
COALESCE to false. Then if mrd.attrs->'value' is null we get a false.
In the process we move any and all negations to outside the COALESCE.
Also apply the same logic to the bib_source filter, not to mention
making it support being negated.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 9ddb169..c0e345b 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -984,6 +984,7 @@ sub flatten {
my $joiner = sprintf(" %s ", ($self->joiner eq '&' ? 'AND' : 'OR'));
# for each dynamic filter, build more of the WHERE clause
for my $filter (@{$self->filters}) {
+ my $NOT = $filter->negate ? 'NOT ' : '';
if (grep { $_ eq $filter->name } @{ $self->QueryParser->dynamic_filters }) {
warn "flatten(): processing dynamic filter ". $filter->name ."\n"
@@ -993,42 +994,40 @@ sub flatten {
$where .= $joiner if $where ne '(';
my @fargs = @{$filter->args};
- my $NOT = $filter->negate ? ' NOT' : '';
my $fname = $filter->name;
$fname = 'item_lang' if $fname eq 'language'; #XXX filter aliases
$where .= sprintf(
- "attrs->'%s'$NOT IN (%s)", $fname,
+ "${NOT}COALESCE((mrd.attrs->'%s') IN (%s), false)", $fname,
join(',', map { $self->QueryParser->quote_value($_) } @fargs)
);
warn "flatten(): filter where => $where\n"
if $self->QueryParser->debug;
} else {
- my $NOT = $filter->negate ? 'NOT ' : '';
switch ($filter->name) {
case 'before' {
if (@{$filter->args} == 1) {
$where .= $joiner if $where ne '(';
- $where .= "$NOT(mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]);
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
}
}
case 'after' {
if (@{$filter->args} == 1) {
$where .= $joiner if $where ne '(';
- $where .= "$NOT(mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]);
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]) . ", false)";
}
}
case 'during' {
if (@{$filter->args} == 1) {
$where .= $joiner if $where ne '(';
- $where .= $self->QueryParser->quote_value($filter->args->[0]) . " ${NOT}BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2')";
+ $where .= "${NOT}COALESCE(" . $self->QueryParser->quote_value($filter->args->[0]) . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2'), false)";
}
}
case 'between' {
if (@{$filter->args} == 2) {
$where .= $joiner if $where ne '(';
- $where .= "(mrd.attrs->'date1') ${NOT}BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]);
+ $where .= "${NOT}COALESCE((mrd.attrs->'date1') BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]) . ", false)";
}
}
case 'container' {
@@ -1096,7 +1095,7 @@ sub flatten {
case 'bib_source' {
if (@{$filter->args} > 0) {
$where .= $joiner if $where ne '(';
- $where .= "bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")";
+ $where .= "${NOT}COALESCE(bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . "), false)";
}
}
}
commit 5aa5e26f4053e7193f7cf8f9b6845f027ce34917
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 11 09:48:39 2012 -0400
Add bib_source filter
Because it would likely be very useful, if only for staff.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 8a13776..9ddb169 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -499,6 +499,7 @@ __PACKAGE__->add_search_filter( 'during' );
__PACKAGE__->add_search_filter( 'statuses' );
__PACKAGE__->add_search_filter( 'locations' );
__PACKAGE__->add_search_filter( 'location_groups', sub { return __PACKAGE__->location_groups_callback(@_) } );
+__PACKAGE__->add_search_filter( 'bib_source' );
__PACKAGE__->add_search_filter( 'site' );
__PACKAGE__->add_search_filter( 'pref_ou' );
__PACKAGE__->add_search_filter( 'lasso' );
@@ -1092,6 +1093,12 @@ sub flatten {
$where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
}
}
+ case 'bib_source' {
+ if (@{$filter->args} > 0) {
+ $where .= $joiner if $where ne '(';
+ $where .= "bre.source IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ")";
+ }
+ }
}
}
}
commit 60a420efd8e043439684af693ca331b023106924
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 11 09:42:38 2012 -0400
Fix empty statuses filter
By adding a check that was overlooked.
Also remove a leftover TODO note and add another test query.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index de68579..8a13776 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -613,7 +613,6 @@ sub toSQL {
my $key = 'm.source';
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->modifiers});
- #TODO: Examine how we want to do limits. And offsets. And other annoying crap like that.
my $core_limit = $self->QueryParser->core_limit || 25000;
my $flat_where = $$flat_plan{where};
@@ -1086,10 +1085,12 @@ sub flatten {
}
}
case 'statuses' {
+ if (@{$filter->args} > 0) {
$where .= $joiner if $where ne '(';
$where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
$where .= $filter->negate ? ' AND ' : ' OR ';
$where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ }
}
}
}
diff --git a/Open-ILS/src/support-scripts/test-scripts/query_tests.pl b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
index c8a2715..042d88d 100755
--- a/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
+++ b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
@@ -26,6 +26,7 @@ my @default_queries = (
'-"keyword1"',
'keyword:"keyword1"',
'keyword:"keyword1" title:"keyword2"',
+ 'keyword locations() statuses()',
# A small set of searches that errored out in a production install
'keyword: subject:Graphical user interfaces (Computer systems) depth(0) subject|topic[Authoring programs]',
'keyword: subject:Assassins New York (State) depth(0) subject|geographic[Buffalo (N.Y.)]',
commit f5a4c11716fbcf5a248dfa69533fbc660e646e0c
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 11 09:21:36 2012 -0400
QueryParser Driver: Use proper table alias
When all atoms are dummy atoms we need the correct table alias.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index cc2304d..de68579 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -876,7 +876,7 @@ sub flatten {
$with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq )";
$from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
} else {
- $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}, (SELECT NULL::tsquery AS tsq ) AS x";
+ $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}, (SELECT NULL::tsquery AS tsq ) AS ${talias}_xq";
}
my @bump_fields;
commit 52d16172cf8eed5b11086b01361cfe9def4dc8be
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Tue Sep 11 09:11:29 2012 -0400
Add some more test queries to query_tester.pl
At least one of which requires a follow-up change to the driver to resolve.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/support-scripts/test-scripts/query_tests.pl b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
index f8d0984..c8a2715 100755
--- a/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
+++ b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
@@ -22,6 +22,10 @@ my @default_queries = (
'(item_type(a)) keyword1 title:keyword2',
'concerto',
'concerto (violin || piano)',
+ '-keyword1',
+ '-"keyword1"',
+ 'keyword:"keyword1"',
+ 'keyword:"keyword1" title:"keyword2"',
# A small set of searches that errored out in a production install
'keyword: subject:Graphical user interfaces (Computer systems) depth(0) subject|topic[Authoring programs]',
'keyword: subject:Assassins New York (State) depth(0) subject|geographic[Buffalo (N.Y.)]',
commit ab9fb958e387a20cfe9fafb6035fb72bc5f1fb3f
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Mon Sep 10 16:47:15 2012 -0400
Remove search.query_parser_fts from schema
Including an upgrade script to drop it.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/sql/Pg/040.schema.asset.sql b/Open-ILS/src/sql/Pg/040.schema.asset.sql
index 7497878..541cff5 100644
--- a/Open-ILS/src/sql/Pg/040.schema.asset.sql
+++ b/Open-ILS/src/sql/Pg/040.schema.asset.sql
@@ -126,7 +126,7 @@ CREATE TABLE asset.opac_visible_copies (
);
COMMENT ON TABLE asset.opac_visible_copies IS $$
Materialized view of copies that are visible in the OPAC, used by
-search.query_parser_fts() to speed up OPAC visibility checks on large
+staged search to speed up OPAC visibility checks on large
databases. Contents are maintained by a set of triggers.
$$;
CREATE INDEX opac_visible_copies_idx1 on asset.opac_visible_copies (record, circ_lib);
diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
index 26d9b53..c0f7f55 100644
--- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
+++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
@@ -30,332 +30,5 @@ CREATE TABLE search.relevance_adjustment (
);
CREATE UNIQUE INDEX bump_once_per_field_idx ON search.relevance_adjustment ( field, bump_type );
-CREATE TYPE search.search_result AS ( id BIGINT, rel NUMERIC, record INT, total INT, checked INT, visible INT, deleted INT, excluded INT );
-CREATE TYPE search.search_args AS ( id INT, field_class TEXT, field_name TEXT, table_alias TEXT, term TEXT, term_type TEXT );
-
-CREATE OR REPLACE FUNCTION search.query_parser_fts (
-
- param_search_ou INT,
- param_depth INT,
- param_query TEXT,
- param_statuses INT[],
- param_locations INT[],
- param_offset INT,
- param_check INT,
- param_limit INT,
- metarecord BOOL,
- staff BOOL,
- param_pref_ou INT DEFAULT NULL
-) RETURNS SETOF search.search_result AS $func$
-DECLARE
-
- current_res search.search_result%ROWTYPE;
- search_org_list INT[];
- luri_org_list INT[];
- tmp_int_list INT[];
-
- check_limit INT;
- core_limit INT;
- core_offset INT;
- tmp_int INT;
-
- core_result RECORD;
- core_cursor REFCURSOR;
- core_rel_query TEXT;
-
- total_count INT := 0;
- check_count INT := 0;
- deleted_count INT := 0;
- visible_count INT := 0;
- excluded_count INT := 0;
-
-BEGIN
-
- check_limit := COALESCE( param_check, 1000 );
- core_limit := COALESCE( param_limit, 25000 );
- core_offset := COALESCE( param_offset, 0 );
-
- -- core_skip_chk := COALESCE( param_skip_chk, 1 );
-
- IF param_search_ou > 0 THEN
- IF param_depth IS NOT NULL THEN
- SELECT array_accum(distinct id) INTO search_org_list FROM actor.org_unit_descendants( param_search_ou, param_depth );
- ELSE
- SELECT array_accum(distinct id) INTO search_org_list FROM actor.org_unit_descendants( param_search_ou );
- END IF;
-
- SELECT array_accum(distinct id) INTO luri_org_list FROM actor.org_unit_ancestors( param_search_ou );
-
- ELSIF param_search_ou < 0 THEN
- SELECT array_accum(distinct org_unit) INTO search_org_list FROM actor.org_lasso_map WHERE lasso = -param_search_ou;
-
- FOR tmp_int IN SELECT * FROM UNNEST(search_org_list) LOOP
- SELECT array_accum(distinct id) INTO tmp_int_list FROM actor.org_unit_ancestors( tmp_int );
- luri_org_list := luri_org_list || tmp_int_list;
- END LOOP;
-
- SELECT array_accum(DISTINCT x.id) INTO luri_org_list FROM UNNEST(luri_org_list) x(id);
-
- ELSIF param_search_ou = 0 THEN
- -- reserved for user lassos (ou_buckets/type='lasso') with ID passed in depth ... hack? sure.
- END IF;
-
- IF param_pref_ou IS NOT NULL THEN
- SELECT array_accum(distinct id) INTO tmp_int_list FROM actor.org_unit_ancestors(param_pref_ou);
- luri_org_list := luri_org_list || tmp_int_list;
- END IF;
-
- OPEN core_cursor FOR EXECUTE param_query;
-
- LOOP
-
- FETCH core_cursor INTO core_result;
- EXIT WHEN NOT FOUND;
- EXIT WHEN total_count >= core_limit;
-
- total_count := total_count + 1;
-
- CONTINUE WHEN total_count NOT BETWEEN core_offset + 1 AND check_limit + core_offset;
-
- check_count := check_count + 1;
-
- PERFORM 1 FROM biblio.record_entry b WHERE NOT b.deleted AND b.id IN ( SELECT * FROM unnest( core_result.records ) );
- IF NOT FOUND THEN
- -- RAISE NOTICE ' % were all deleted ... ', core_result.records;
- deleted_count := deleted_count + 1;
- CONTINUE;
- END IF;
-
- PERFORM 1
- FROM biblio.record_entry b
- JOIN config.bib_source s ON (b.source = s.id)
- WHERE s.transcendant
- AND b.id IN ( SELECT * FROM unnest( core_result.records ) );
-
- IF FOUND THEN
- -- RAISE NOTICE ' % were all transcendant ... ', core_result.records;
- visible_count := visible_count + 1;
-
- current_res.id = core_result.id;
- current_res.rel = core_result.rel;
-
- tmp_int := 1;
- IF metarecord THEN
- SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
- END IF;
-
- IF tmp_int = 1 THEN
- current_res.record = core_result.records[1];
- ELSE
- current_res.record = NULL;
- END IF;
-
- RETURN NEXT current_res;
-
- CONTINUE;
- END IF;
-
- PERFORM 1
- FROM asset.call_number cn
- JOIN asset.uri_call_number_map map ON (map.call_number = cn.id)
- JOIN asset.uri uri ON (map.uri = uri.id)
- WHERE NOT cn.deleted
- AND cn.label = '##URI##'
- AND uri.active
- AND ( param_locations IS NULL OR array_upper(param_locations, 1) IS NULL )
- AND cn.record IN ( SELECT * FROM unnest( core_result.records ) )
- AND cn.owning_lib IN ( SELECT * FROM unnest( luri_org_list ) )
- LIMIT 1;
-
- IF FOUND THEN
- -- RAISE NOTICE ' % have at least one URI ... ', core_result.records;
- visible_count := visible_count + 1;
-
- current_res.id = core_result.id;
- current_res.rel = core_result.rel;
-
- tmp_int := 1;
- IF metarecord THEN
- SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
- END IF;
-
- IF tmp_int = 1 THEN
- current_res.record = core_result.records[1];
- ELSE
- current_res.record = NULL;
- END IF;
-
- RETURN NEXT current_res;
-
- CONTINUE;
- END IF;
-
- IF param_statuses IS NOT NULL AND array_upper(param_statuses, 1) > 0 THEN
-
- PERFORM 1
- FROM asset.call_number cn
- JOIN asset.copy cp ON (cp.call_number = cn.id)
- WHERE NOT cn.deleted
- AND NOT cp.deleted
- AND cp.status IN ( SELECT * FROM unnest( param_statuses ) )
- AND cn.record IN ( SELECT * FROM unnest( core_result.records ) )
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
- PERFORM 1
- FROM biblio.peer_bib_copy_map pr
- JOIN asset.copy cp ON (cp.id = pr.target_copy)
- WHERE NOT cp.deleted
- AND cp.status IN ( SELECT * FROM unnest( param_statuses ) )
- AND pr.peer_record IN ( SELECT * FROM unnest( core_result.records ) )
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
- -- RAISE NOTICE ' % and multi-home linked records were all status-excluded ... ', core_result.records;
- excluded_count := excluded_count + 1;
- CONTINUE;
- END IF;
- END IF;
-
- END IF;
-
- IF param_locations IS NOT NULL AND array_upper(param_locations, 1) > 0 THEN
-
- PERFORM 1
- FROM asset.call_number cn
- JOIN asset.copy cp ON (cp.call_number = cn.id)
- WHERE NOT cn.deleted
- AND NOT cp.deleted
- AND cp.location IN ( SELECT * FROM unnest( param_locations ) )
- AND cn.record IN ( SELECT * FROM unnest( core_result.records ) )
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
- PERFORM 1
- FROM biblio.peer_bib_copy_map pr
- JOIN asset.copy cp ON (cp.id = pr.target_copy)
- WHERE NOT cp.deleted
- AND cp.location IN ( SELECT * FROM unnest( param_locations ) )
- AND pr.peer_record IN ( SELECT * FROM unnest( core_result.records ) )
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
- -- RAISE NOTICE ' % and multi-home linked records were all copy_location-excluded ... ', core_result.records;
- excluded_count := excluded_count + 1;
- CONTINUE;
- END IF;
- END IF;
-
- END IF;
-
- IF staff IS NULL OR NOT staff THEN
-
- PERFORM 1
- FROM asset.opac_visible_copies
- WHERE circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- AND record IN ( SELECT * FROM unnest( core_result.records ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
- PERFORM 1
- FROM biblio.peer_bib_copy_map pr
- JOIN asset.opac_visible_copies cp ON (cp.copy_id = pr.target_copy)
- WHERE cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- AND pr.peer_record IN ( SELECT * FROM unnest( core_result.records ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
-
- -- RAISE NOTICE ' % and multi-home linked records were all visibility-excluded ... ', core_result.records;
- excluded_count := excluded_count + 1;
- CONTINUE;
- END IF;
- END IF;
-
- ELSE
-
- PERFORM 1
- FROM asset.call_number cn
- JOIN asset.copy cp ON (cp.call_number = cn.id)
- WHERE NOT cn.deleted
- AND NOT cp.deleted
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- AND cn.record IN ( SELECT * FROM unnest( core_result.records ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
-
- PERFORM 1
- FROM biblio.peer_bib_copy_map pr
- JOIN asset.copy cp ON (cp.id = pr.target_copy)
- WHERE NOT cp.deleted
- AND cp.circ_lib IN ( SELECT * FROM unnest( search_org_list ) )
- AND pr.peer_record IN ( SELECT * FROM unnest( core_result.records ) )
- LIMIT 1;
-
- IF NOT FOUND THEN
-
- PERFORM 1
- FROM asset.call_number cn
- JOIN asset.copy cp ON (cp.call_number = cn.id)
- WHERE cn.record IN ( SELECT * FROM unnest( core_result.records ) )
- AND NOT cp.deleted
- LIMIT 1;
-
- IF FOUND THEN
- -- RAISE NOTICE ' % and multi-home linked records were all visibility-excluded ... ', core_result.records;
- excluded_count := excluded_count + 1;
- CONTINUE;
- END IF;
- END IF;
-
- END IF;
-
- END IF;
-
- visible_count := visible_count + 1;
-
- current_res.id = core_result.id;
- current_res.rel = core_result.rel;
-
- tmp_int := 1;
- IF metarecord THEN
- SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
- END IF;
-
- IF tmp_int = 1 THEN
- current_res.record = core_result.records[1];
- ELSE
- current_res.record = NULL;
- END IF;
-
- RETURN NEXT current_res;
-
- IF visible_count % 1000 = 0 THEN
- -- RAISE NOTICE ' % visible so far ... ', visible_count;
- END IF;
-
- END LOOP;
-
- current_res.id = NULL;
- current_res.rel = NULL;
- current_res.record = NULL;
- current_res.total = total_count;
- current_res.checked = check_count;
- current_res.deleted = deleted_count;
- current_res.visible = visible_count;
- current_res.excluded = excluded_count;
-
- CLOSE core_cursor;
-
- RETURN NEXT current_res;
-
-END;
-$func$ LANGUAGE PLPGSQL;
-
COMMIT;
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql
new file mode 100644
index 0000000..ee76bd1
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.drop.query_parser_fts.sql
@@ -0,0 +1,3 @@
+DROP FUNCTION IF EXISTS search.query_parser_fts(INT,INT,TEXT,INT[],INT[],INT,INT,INT,BOOL,BOOL,INT);
+DROP TYPE IF EXISTS search.search_result;
+DROP TYPE IF EXISTS search.search_args;
commit 613a6da032ab0d177421c36fe07d1d1dd9c6922c
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Mon Sep 10 13:54:55 2012 -0400
Quick script for pushing queries into search
Several of the defaults are designed to trigger bad SQL generation.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/support-scripts/test-scripts/query_tests.pl b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
new file mode 100755
index 0000000..f8d0984
--- /dev/null
+++ b/Open-ILS/src/support-scripts/test-scripts/query_tests.pl
@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+require '../oils_header.pl';
+use strict; use warnings;
+use OpenSRF::EX qw(:try);
+use OpenSRF::AppSession;
+use Getopt::Long;
+use Data::Dumper;
+
+my $config = '/openils/conf/opensrf_core.xml';
+my $debug = 0;
+my @default_queries = (
+ 'keyword1',
+ 'keyword1 || keyword2',
+ '(keyword1) || keyword2',
+ 'keyword1 || (keyword2)',
+ '(keyword1) || (keyword2)',
+ 'keyword item_type(a)',
+ '(item_type(a)) keyword1',
+ 'keyword1 item_type(a) title:keyword2',
+ 'keyword1 (item_type(a)) title:keyword2',
+ 'item_type(a) keyword1 title:keyword2',
+ '(item_type(a)) keyword1 title:keyword2',
+ 'concerto',
+ 'concerto (violin || piano)',
+# A small set of searches that errored out in a production install
+ 'keyword: subject:Graphical user interfaces (Computer systems) depth(0) subject|topic[Authoring programs]',
+ 'keyword: subject:Assassins New York (State) depth(0) subject|geographic[Buffalo (N.Y.)]',
+ 'keyword: author: Niggeman Indifilm (Firm) depth(0) subject|geographic[Mars (Planet)]',
+ 'keyword: subject:Los Angeles (Calif.) Juvenile fiction. depth(0) subject|geographic[Los Angeles (Calif.)]',
+ 'keyword: subject:Los Angeles (Calif.) depth(0) subject|geographic[California] subject|name[Faulkner, William 1897-1962]',
+ 'keyword: subject:Thrillers (Motion pictures, television, etc.) depth(0) subject|topic[Action and adventure films]',
+ 'keyword: author: Brilliance Audio (Firm) depth(0) subject|topic[Man-woman relationships]',
+ 'keyword: subject:Rhodenbarr, Bernie (Fictitious character) depth(0) subject|geographic[England] subject|topic[Audiocassettes]',
+ 'keyword: subject:Burgett, Donald R. (Donald Robert), depth(0) subject|geographic[Netherlands]',
+ 'keyword: author: 2 Entertain (Firm) depth(0) subject|geographic[England] subject|geographic[Nottingham (England)]',
+# Selection from the query_parser.pl script
+ '#available title: foo bar* || (-baz || (subject:"1900'.
+ '-1910 junk" "and another thing" se:stuff #available '.
+ 'statuses(0,7,12))) && && && au:malarky || au|'.
+ 'corporate|personal:gonzo && dc.identifier:+123456789X'.
+ ' dc.contributor=rowling #metarecord estimation_'.
+ 'strategy(exclusion) item_type(a, t) item_form(d) '.
+ 'bib.subjectTitle=potter bib.subjectName=harry '.
+ 'keyword|mapscale:1:250000',
+ 'concerto #available filter_group_entry(1,2,3) filter_group_entry(4,5)',
+ 'concerto || filter_group_entry(4) || filter_group_entry(3)',
+ 'concerto (audience(a) || (item_type(a) && item_form(b)))',
+ 'concerto || (piano && (item_type(a) || audience(a)))',
+ '(concerto item_type(a)) || (piano item_type(b))',
+ 'audience(a) (concerto || item_type(a) || (piano music item_form(b)))',
+ 'concerto && (item_type(a) || piano) && (item_form(b) || music)',
+ 'concerto && (piano || item_type(a)) && (music || item_form(b))',
+
+);
+
+my @queries;
+
+GetOptions(
+ 'config=s' => \$config,
+ 'debug' => \$debug,
+ 'query=s' => \@queries,
+);
+osrf_connect($config); # connect to jabber
+
+ at queries = @default_queries unless @queries;
+
+my $ses = OpenSRF::AppSession->create("open-ils.search");
+$ses->connect;
+print "Running Queries\n";
+foreach (@queries) {
+ try {
+ my $req = $ses->request('open-ils.search.biblio.multiclass.query', {}, $_, 0);
+ my $stat = $req->gather(1);
+ print "Query $_ returned " . $stat->{count} . " results\n";
+ } catch Error with {
+ print "ERROR ON QUERY: $_\n";
+ };
+}
+print "Done\n";
+$ses->disconnect;
+
+
commit 6d8872cf120caf67ad6f65995b2c5155fa5ab652
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Fri Sep 7 16:07:26 2012 -0400
Remove dependence on search.query_parser_fts proc
We do this by moving everything into the QueryParser driver, then telling
the metabib layer to not call it anymore.
In the process we remove the "superpage" checks, instead just getting the
entire result set directly from the DB.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example
index ae7da96..91243b1 100644
--- a/Open-ILS/examples/opensrf.xml.example
+++ b/Open-ILS/examples/opensrf.xml.example
@@ -502,22 +502,6 @@ vim:et:ts=4:sw=4:
<use_staged_search>true</use_staged_search>
<!--
- For staged search, we estimate hits based on inclusion or exclusion.
-
- Valid settings:
- inclusion - visible ratio on superpage
- exclusion - excluded ratio on superpage
- delete_adjusted_inclusion - included ratio on superpage, ratio adjusted by deleted count
- delete_adjusted_exclusion - excluded ratio on superpage, ratio adjusted by deleted count
-
- Under normal circumstances, inclusion is the best strategy, and both delete_adjusted variants
- will return the same value +/- 1. The exclusion strategy is the original, and works well
- when there are few deleted or excluded records, in other words, when the superpage is not
- sparsely populated with visible records.
- -->
- <estimation_strategy>inclusion</estimation_strategy>
-
- <!--
Evergreen uses a cover density algorithm for calculating relative ranking of matches. There
are several tuning parameters and options available. By default, no document length normalization
is applied. From the Postgres documentation on ts_rank_cd() (the function used by Evergreen):
@@ -576,11 +560,8 @@ vim:et:ts=4:sw=4:
-->
<default_preferred_language_weight>5</default_preferred_language_weight>
- <!-- Baseline number of records to check for hit estimation. -->
- <superpage_size>1000</superpage_size>
-
- <!-- How many superpages to consider for searching overall. -->
- <max_superpages>10</max_superpages>
+ <!-- How many search results to return. Defaults to superpage_size * max_superpages, if they are defined and it isn't. -->
+ <max_search_results>10000</max_search_results>
<!-- zip code database file -->
<!--<zips_file>LOCALSTATEDIR/data/zips.txt</zips_file>-->
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
index 865674b..f310a7e 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
@@ -36,8 +36,7 @@ my $pfx = "open-ils.search_";
my $cache;
my $cache_timeout;
-my $superpage_size;
-my $max_superpages;
+my $max_search_results;
sub initialize {
$cache = OpenSRF::Utils::Cache->new('global');
@@ -45,14 +44,17 @@ sub initialize {
$cache_timeout = $sclient->config_value(
"apps", "open-ils.search", "app_settings", "cache_timeout" ) || 300;
- $superpage_size = $sclient->config_value(
+ my $superpage_size = $sclient->config_value(
"apps", "open-ils.search", "app_settings", "superpage_size" ) || 500;
- $max_superpages = $sclient->config_value(
+ my $max_superpages = $sclient->config_value(
"apps", "open-ils.search", "app_settings", "max_superpages" ) || 20;
+ $max_search_results = $sclient->config_value(
+ "apps", "open-ils.search", "app_settings", "max_search_results" ) || ($superpage_size * $max_superpages);
+
$logger->info("Search cache timeout is $cache_timeout, ".
- " superpage_size is $superpage_size, max_superpages is $max_superpages");
+ " max_search_results is $max_search_results");
}
@@ -1269,24 +1271,8 @@ sub staged_search {
$user_offset = ($user_offset >= 0) ? $user_offset : 0;
$user_limit = ($user_limit >= 0) ? $user_limit : 10;
-
- # we're grabbing results on a per-superpage basis, which means the
- # limit and offset should coincide with superpage boundaries
- $search_hash->{offset} = 0;
- $search_hash->{limit} = $superpage_size;
-
- # force a well-known check_limit
- $search_hash->{check_limit} = $superpage_size;
- # restrict total tested to superpage size * number of superpages
- $search_hash->{core_limit} = $superpage_size * $max_superpages;
-
- # Set the configured estimation strategy, defaults to 'inclusion'.
- my $estimation_strategy = OpenSRF::Utils::SettingsClient
- ->new
- ->config_value(
- apps => 'open-ils.search', app_settings => 'estimation_strategy'
- ) || 'inclusion';
- $search_hash->{estimation_strategy} = $estimation_strategy;
+ # restrict DB query to our max results
+ $search_hash->{core_limit} = $max_search_results;
# pull any existing results from the cache
my $key = search_cache_key($method, $search_hash);
@@ -1296,126 +1282,67 @@ sub staged_search {
# keep retrieving results until we find enough to
# fulfill the user-specified limit and offset
my $all_results = [];
- my $page; # current superpage
- my $est_hit_count = 0;
- my $current_page_summary = {};
- my $global_summary = {checked => 0, visible => 0, excluded => 0, deleted => 0, total => 0};
- my $is_real_hit_count = 0;
+
+ my $results;
+ my $summary;
my $new_ids = [];
- for($page = 0; $page < $max_superpages; $page++) {
+ if($cache_data->{summary}) {
+ # this window of results is already cached
+ $logger->debug("staged search: found cached results");
+ $summary = $cache_data->{summary};
+ $results = $cache_data->{results};
- my $data = $cache_data->{$page};
- my $results;
- my $summary;
+ } else {
+ # retrieve the window of results from the database
+ $logger->debug("staged search: fetching results from the database");
+ my $start = time;
+ $results = $U->storagereq($method, %$search_hash);
+ $search_duration = time - $start;
+ $summary = shift(@$results) if $results;
+
+ unless($summary) {
+ $logger->info("search timed out: duration=$search_duration: params=".
+ OpenSRF::Utils::JSON->perl2JSON($search_hash));
+ return {count => 0};
+ }
- $logger->debug("staged search: analyzing superpage $page");
+ $logger->info("staged search: DB call took $search_duration seconds and returned ".scalar(@$results)." rows, including summary");
- if($data) {
- # this window of results is already cached
- $logger->debug("staged search: found cached results");
- $summary = $data->{summary};
- $results = $data->{results};
+ my $hc = $summary->{visible};
+ if($hc == 0) {
+ $logger->info("search returned 0 results: duration=$search_duration: params=".
+ OpenSRF::Utils::JSON->perl2JSON($search_hash));
+ }
+ # Create backwards-compatible result structures
+ if($IAmMetabib) {
+ $results = [map {[$_->{id}, $_->{rel}, $_->{record}]} @$results];
} else {
- # retrieve the window of results from the database
- $logger->debug("staged search: fetching results from the database");
- $search_hash->{skip_check} = $page * $superpage_size;
- my $start = time;
- $results = $U->storagereq($method, %$search_hash);
- $search_duration = time - $start;
- $summary = shift(@$results) if $results;
-
- unless($summary) {
- $logger->info("search timed out: duration=$search_duration: params=".
- OpenSRF::Utils::JSON->perl2JSON($search_hash));
- return {count => 0};
- }
-
- $logger->info("staged search: DB call took $search_duration seconds and returned ".scalar(@$results)." rows, including summary");
-
- my $hc = $summary->{estimated_hit_count} || $summary->{visible};
- if($hc == 0) {
- $logger->info("search returned 0 results: duration=$search_duration: params=".
- OpenSRF::Utils::JSON->perl2JSON($search_hash));
- }
-
- # Create backwards-compatible result structures
- if($IAmMetabib) {
- $results = [map {[$_->{id}, $_->{rel}, $_->{record}]} @$results];
- } else {
- $results = [map {[$_->{id}]} @$results];
- }
-
- push @$new_ids, grep {defined($_)} map {$_->[0]} @$results;
- $results = [grep {defined $_->[0]} @$results];
- cache_staged_search_page($key, $page, $summary, $results) if $docache;
+ $results = [map {[$_->{id}]} @$results];
}
- tag_circulated_records($search_hash->{authtoken}, $results, $IAmMetabib)
- if $search_hash->{tag_circulated_records} and $search_hash->{authtoken};
-
- $current_page_summary = $summary;
-
- # add the new set of results to the set under construction
- push(@$all_results, @$results);
-
- my $current_count = scalar(@$all_results);
-
- $est_hit_count = $summary->{estimated_hit_count} || $summary->{visible}
- if $page == 0;
-
- $logger->debug("staged search: located $current_count, with estimated hits=".
- $summary->{estimated_hit_count}." : visible=".$summary->{visible}.", checked=".$summary->{checked});
+ push @$new_ids, grep {defined($_)} map {$_->[0]} @$results;
+ $results = [grep {defined $_->[0]} @$results];
+ cache_staged_search($key, $summary, $results) if $docache;
+ }
- if (defined($summary->{estimated_hit_count})) {
- foreach (qw/ checked visible excluded deleted /) {
- $global_summary->{$_} += $summary->{$_};
- }
- $global_summary->{total} = $summary->{total};
- }
+ tag_circulated_records($search_hash->{authtoken}, $results, $IAmMetabib)
+ if $search_hash->{tag_circulated_records} and $search_hash->{authtoken};
- # we've found all the possible hits
- last if $current_count == $summary->{visible}
- and not defined $summary->{estimated_hit_count};
+ # add the new set of results to the set under construction
+ push(@$all_results, @$results);
- # we've found enough results to satisfy the requested limit/offset
- last if $current_count >= ($user_limit + $user_offset);
+ my $current_count = scalar(@$all_results);
- # we've scanned all possible hits
- if($summary->{checked} < $superpage_size) {
- $est_hit_count = scalar(@$all_results);
- # we have all possible results in hand, so we know the final hit count
- $is_real_hit_count = 1;
- last;
- }
- }
+ $logger->debug("staged search: located $current_count, visible=".$summary->{visible});
my @results = grep {defined $_} @$all_results[$user_offset..($user_offset + $user_limit - 1)];
- # refine the estimate if we have more than one superpage
- if ($page > 0 and not $is_real_hit_count) {
- if ($global_summary->{checked} >= $global_summary->{total}) {
- $est_hit_count = $global_summary->{visible};
- } else {
- my $updated_hit_count = $U->storagereq(
- 'open-ils.storage.fts_paging_estimate',
- $global_summary->{checked},
- $global_summary->{visible},
- $global_summary->{excluded},
- $global_summary->{deleted},
- $global_summary->{total}
- );
- $est_hit_count = $updated_hit_count->{$estimation_strategy};
- }
- }
-
$conn->respond_complete(
{
- count => $est_hit_count,
+ count => $summary->{visible},
core_limit => $search_hash->{core_limit},
- superpage_size => $search_hash->{check_limit},
- superpage_summary => $current_page_summary,
facet_key => $facet_key,
ids => \@results
}
@@ -1582,18 +1509,15 @@ sub cache_facets {
$cache->put_cache($key, $data, $cache_timeout);
}
-sub cache_staged_search_page {
+sub cache_staged_search {
# puts this set of results into the cache
- my($key, $page, $summary, $results) = @_;
- my $data = $cache->get_cache($key);
- $data ||= {};
- $data->{$page} = {
+ my($key, $summary, $results) = @_;
+ my $data = {
summary => $summary,
results => $results
};
- $logger->info("staged search: cached with key=$key, superpage=$page, estimated=".
- $summary->{estimated_hit_count}.", visible=".$summary->{visible});
+ $logger->info("staged search: cached with key=$key, visible=".$summary->{visible});
$cache->put_cache($key, $data, $cache_timeout);
}
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 1b95080..cc2304d 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -45,6 +45,24 @@ sub filter_group_entry_callback {
);
}
+sub location_groups_callback {
+ my ($invocant, $self, $struct, $filter, $params, $negate) = @_;
+
+ return sprintf(' %slocations(%s)',
+ $negate ? '-' : '',
+ join(
+ ',',
+ map {
+ $_->location
+ } @{
+ OpenILS::Utils::CStoreEditor
+ ->new
+ ->search_asset_copy_location_group_map({ lgroup => $params })
+ }
+ )
+ );
+}
+
sub format_callback {
my ($invocant, $self, $struct, $filter, $params, $negate) = @_;
@@ -477,10 +495,10 @@ __PACKAGE__->add_search_filter( 'after' );
__PACKAGE__->add_search_filter( 'between' );
__PACKAGE__->add_search_filter( 'during' );
-# used by layers above this
+# various filters for limiting in various ways
__PACKAGE__->add_search_filter( 'statuses' );
__PACKAGE__->add_search_filter( 'locations' );
-__PACKAGE__->add_search_filter( 'location_groups' );
+__PACKAGE__->add_search_filter( 'location_groups', sub { return __PACKAGE__->location_groups_callback(@_) } );
__PACKAGE__->add_search_filter( 'site' );
__PACKAGE__->add_search_filter( 'pref_ou' );
__PACKAGE__->add_search_filter( 'lasso' );
@@ -527,8 +545,9 @@ use base 'QueryParser::query_plan';
use OpenSRF::Utils::Logger qw($logger);
use Data::Dumper;
use OpenILS::Application::AppUtils;
+use OpenILS::Utils::CStoreEditor;
my $apputils = "OpenILS::Application::AppUtils";
-
+my $editor = OpenILS::Utils::CStoreEditor->new;
sub toSQL {
my $self = shift;
@@ -543,6 +562,7 @@ sub toSQL {
$filters{$col} = $filter->args->[0];
}
}
+ $self->new_filter( statuses => [0,7,12] ) if ($self->find_modifier('available'));
$self->QueryParser->superpage($filters{superpage}) if ($filters{superpage});
$self->QueryParser->superpage_size($filters{superpage_size}) if ($filters{superpage_size});
@@ -553,7 +573,8 @@ sub toSQL {
my $flat_plan = $self->flatten;
# generate the relevance ranking
- my $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1";
+ my $rel = '1'; # Default to something simple in case rank_list is empty.
+ $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1" if (@{$$flat_plan{rank_list}});
# find any supplied sort option
my ($sort_filter) = $self->find_filter('sort');
@@ -570,8 +591,6 @@ sub toSQL {
}
$rel = "1.0/($rel)::NUMERIC";
- my $mra_join = 'INNER JOIN metabib.record_attr mrd ON m.source = mrd.id';
-
my $rank = $rel;
my $desc = 'ASC';
@@ -594,6 +613,7 @@ sub toSQL {
my $key = 'm.source';
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->modifiers});
+ #TODO: Examine how we want to do limits. And offsets. And other annoying crap like that.
my $core_limit = $self->QueryParser->core_limit || 25000;
my $flat_where = $$flat_plan{where};
@@ -602,29 +622,194 @@ sub toSQL {
} else {
$flat_where = "AND $flat_where";
}
- my $with = $$flat_plan{with};
- $with= "\nWITH $with" if $with;
- # Need an array for query parser db function; this gives a better plan
- # than the ARRAY_AGG(DISTINCT m.source) option as of PostgreSQL 9.1
- my $agg_records = 'ARRAY[m.source] AS records';
+ my $site = $self->find_filter('site');
+ if ($site && $site->args) {
+ $site = $site->args->[0];
+ if ($site && $site !~ /^(-)?\d+$/) {
+ my $search = $editor->search_actor_org_unit({ shortname => $site });
+ $site = @$search[0]->id if($search && @$search);
+ $site = undef unless ($search);
+ }
+ } else {
+ $site = undef;
+ }
+ my $lasso = $self->find_filter('lasso');
+ if ($lasso && $lasso->args) {
+ $lasso = $lasso->args->[0];
+ if ($lasso && $lasso !~ /^\d+$/) {
+ my $search = $editor->search_actor_org_lasso({ name => $lasso });
+ $lasso = @$search[0]->id if($search && @$search);
+ $lasso = undef unless ($search);
+ }
+ } else {
+ $lasso = undef;
+ }
+ my $depth = $self->find_filter('depth');
+ if ($depth && $depth->args) {
+ $depth = $depth->args->[0];
+ if ($depth && $depth !~ /^\d+$/) {
+ # This *is* what metabib.pm has been doing....but it makes no sense to me. :/
+ # Should this be looking up the depth of the OU type on the OU in question?
+ my $search = $editor->search_actor_org_unit([{ name => $depth },{ opac_label => $depth }]);
+ $depth = @$search[0]->id if($search && @$search);
+ $depth = undef unless($search);
+ }
+ } else {
+ $depth = undef;
+ }
+ my $pref_ou = $self->find_filter('pref_ou');
+ if ($pref_ou && $pref_ou->args) {
+ $pref_ou = $pref_ou->args->[0];
+ if ($pref_ou && $pref_ou !~ /^(-)?\d+$/) {
+ my $search = $editor->search_actor_org_unit({ shortname => $pref_ou });
+ $pref_ou = @$search[0]->id if($search && @$search);
+ $pref_ou = undef unless ($search);
+ }
+ } else {
+ $pref_ou = undef;
+ }
+
+ # Supposedly at some point a site of 0 and a depth will equal user lasso id.
+ # We need OU buckets before that happens. 'my_lasso' is, I believe, the target filter for it.
+
+ $site = -$lasso if ($lasso);
+
+ # Default to the top of the org tree if we have nothing else. This would need tweaking for the user lasso bit.
+ if (!$site) {
+ my $search = $editor->search_actor_org_unit({ parent_ou => undef });
+ $site = @$search[0]->id if ($search);
+ }
+
+ my $depth_check = '';
+ $depth_check = ", $depth" if ($depth);
+
+ my $with = '';
+ $with .= " search_org_list AS (\n";
+ if ($site < 0) {
+ # Lasso!
+ $lasso = -$site;
+ $with .= " SELECT DISTINCT org_unit from actor.org_lasso_map WHERE lasso = $lasso\n";
+ } elsif ($site > 0) {
+ $with .= " SELECT DISTINCT id FROM actor.org_unit_descendants($site$depth_check)\n";
+ } else {
+ # Placeholder for user lasso stuff.
+ }
+ $with .= " ),\n";
+ $with .= " luri_org_list AS (\n";
+ if ($site < 0) {
+ # We can re-use the lasso var, we already updated it above.
+ $with .= " SELECT DISTINCT (actor.org_unit_ancestors(org_unit)).id from actor.org_lasso_map WHERE lasso = $lasso\n";
+ } elsif ($site > 0) {
+ $with .= " SELECT DISTINCT id FROM actor.org_unit_ancestors($site)\n";
+ } else {
+ # Placeholder for user lasso stuff.
+ }
+ if ($pref_ou) {
+ $with .= " UNION\n";
+ $with .= " SELECT DISTINCT id FROM actor.org_unit_ancestors($pref_ou)\n";
+ }
+ $with .= " )";
+ $with .= ",\n " . $$flat_plan{with} if ($$flat_plan{with});
+
+ # Limit stuff
+ my $limit_where = <<" SQL";
+-- Filter records based on visibility
+ AND (
+ cbs.transcendant IS TRUE
+ OR
+ EXISTS(
+ SELECT 1 FROM asset.call_number acn
+ JOIN asset.uri_call_number_map aucnm ON acn.id = aucnm.call_number
+ JOIN asset.uri uri ON aucnm.uri = uri.id
+ WHERE NOT acn.deleted AND uri.active AND acn.record = m.source AND acn.owning_lib IN (
+ SELECT * FROM luri_org_list
+ )
+ LIMIT 1
+ )
+ OR
+ SQL
+ if ($self->find_modifier('staff')) {
+ $limit_where .= <<" SQL";
+ EXISTS(
+ SELECT 1 FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ WHERE NOT cn.deleted
+ AND NOT cp.deleted
+ AND cp.circ_lib IN ( SELECT * FROM search_org_list )
+ AND cn.record = m.source
+ LIMIT 1
+ )
+ OR
+ EXISTS(
+ SELECT 1 FROM biblio.peer_bib_copy_map pr
+ JOIN asset.copy cp ON (cp.id = pr.target_copy)
+ WHERE NOT cp.deleted
+ AND cp.circ_lib IN ( SELECT * FROM search_org_list )
+ AND pr.peer_record = m.source
+ LIMIT 1
+ )
+ OR (
+ NOT EXISTS(
+ SELECT 1 FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ WHERE cn.record = m.source
+ AND NOT cp.deleted
+ LIMIT 1
+ )
+ AND
+ NOT EXISTS(
+ SELECT 1 FROM biblio.peer_bib_copy_map pr
+ JOIN asset.copy cp ON (cp.id = pr.target_copy)
+ WHERE NOT cp.deleted
+ AND pr.peer_record = m.source
+ LIMIT 1
+ )
+ )
+ SQL
+ } else {
+ $limit_where .= <<" SQL";
+ EXISTS(
+ SELECT 1 FROM asset.opac_visible_copies
+ WHERE circ_lib IN ( SELECT * FROM search_org_list )
+ AND record = m.source
+ LIMIT 1
+ )
+ OR
+ EXISTS(
+ SELECT 1 FROM biblio.peer_bib_copy_map pr
+ JOIN asset.opac_visible_copies cp ON (cp.copy_id = pr.target_copy)
+ WHERE cp.circ_lib IN ( SELECT * FROM search_org_list )
+ AND pr.peer_record = m.source
+ LIMIT 1
+ )
+ SQL
+ }
+ $limit_where .= " )";
+
+ # For single records we want the record id
+ # For metarecords we want NULL or the only record ID.
+ my $agg_record = 'm.source AS record';
if ($key =~ /metarecord/) {
- # metarecord searches still require the ARRAY_AGG approach
- $agg_records = 'ARRAY_AGG(DISTINCT m.source) AS records';
+ $agg_record = 'CASE WHEN COUNT(DISTINCT m.source) = 1 THEN FIRST(m.source) ELSE NULL END AS record';
}
my $sql = <<SQL;
+WITH
$with
SELECT $key AS id,
- $agg_records,
+ $agg_record,
$rel AS rel,
$rank AS rank,
FIRST(mrd.attrs->'date1') AS tie_break
FROM metabib.metarecord_source_map m
$$flat_plan{from}
- $mra_join
+ INNER JOIN metabib.record_attr mrd ON m.source = mrd.id
+ INNER JOIN biblio.record_entry bre ON m.source = bre.id
+ LEFT JOIN config.bib_source cbs ON bre.source = cbs.id
WHERE 1=1
$flat_where
+ $limit_where
GROUP BY 1
ORDER BY 4 $desc $nullpos, 5 DESC $nullpos, 3 DESC
LIMIT $core_limit
@@ -758,9 +943,12 @@ sub flatten {
"AND SUBSTRING(${talias}.value,1,1024) IN (" . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n${spc}${spc}".
"AND ${talias}.field IN (". join(',', @field_ids) . ")\n${spc})";
- if ($join_type != 'INNER') {
+ if ($join_type ne 'INNER') {
my $NOT = $node->negate ? '' : ' NOT';
$where .= "${talias}.id IS$NOT NULL";
+ } elsif ($where ne '(') {
+ # Strip extra joiner
+ $where =~ s/\s(AND|OR)\s$//;
}
} else {
@@ -872,7 +1060,12 @@ sub flatten {
my ($u,$e) = $apputils->checksesperm($token) if ($token);
$perm_join = ' OR c.owner = ' . $u->id if ($u && !$e);
$where .= $joiner if $where ne '(';
- $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)"
+ $where .= '(' if $class eq 'copy';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)";
+ }
+ if ($class eq 'copy') {
+ my $subjoiner = $filter->negate ? ' AND ' : ' OR ';
+ $where .= "$subjoiner${NOT}EXISTS(SELECT 1 FROM container.copy_bucket_item ci JOIN container.copy_bucket c ON (c.id = ci.bucket) JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy WHERE c.btype = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND pr.peer_record = m.source LIMIT 1))";
}
}
}
@@ -881,10 +1074,23 @@ sub flatten {
my $key = 'm.source';
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers});
$where .= $joiner if $where ne '(';
- $where .= 'NOT ' if $filter->negate;
- $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args}) . ')';
+ $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{$filter->args}) . ')';
}
}
+ case 'locations' {
+ if (@{$filter->args} > 0) {
+ $where .= $joiner if $where ne '(';
+ $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
+ $where .= $filter->negate ? ' AND ' : ' OR ';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ }
+ }
+ case 'statuses' {
+ $where .= $joiner if $where ne '(';
+ $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)";
+ $where .= $filter->negate ? ' AND ' : ' OR ';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))";
+ }
}
}
}
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
index 722c6c5..3241bc3 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm
@@ -2929,7 +2929,7 @@ sub query_parser_fts {
# parse the query and supply any query-level %arg-based defaults
- # we expect, and make use of, query, superpage, superpage_size, debug and core_limit args
+ # we expect, and make use of, query, debug and core_limit args
my $query = $parser->new( %args )->parse;
my $config = OpenSRF::Utils::SettingsClient->new();
@@ -2973,194 +2973,15 @@ sub query_parser_fts {
}
}
- # gather the site, if one is specified, defaulting to the in-query version
- my $ou = $args{org_unit};
- if (my ($filter) = $query->parse_tree->find_filter('site')) {
- $ou = $filter->args->[0] if (@{$filter->args});
- }
- $ou = actor::org_unit->search( { shortname => $ou } )->next->id if ($ou and $ou !~ /^(-)?\d+$/);
-
- # gather lasso, as with $ou
- my $lasso = $args{lasso};
- if (my ($filter) = $query->parse_tree->find_filter('lasso')) {
- $lasso = $filter->args->[0] if (@{$filter->args});
- }
- $lasso = actor::org_lasso->search( { name => $lasso } )->next->id if ($lasso and $lasso !~ /^\d+$/);
- $lasso = -$lasso if ($lasso);
-
-
-# # XXX once we have org_unit containers, we can make user-defined lassos .. WHEEE
-# # gather user lasso, as with $ou and lasso
-# my $mylasso = $args{my_lasso};
-# if (my ($filter) = $query->parse_tree->find_filter('my_lasso')) {
-# $mylasso = $filter->args->[0] if (@{$filter->args});
-# }
-# $mylasso = actor::org_unit->search( { name => $mylasso } )->next->id if ($mylasso and $mylasso !~ /^\d+$/);
-
-
- # if we have a lasso, go with that, otherwise ... ou
- $ou = $lasso if ($lasso);
-
- # gather the preferred OU, if one is specified, as with $ou
- my $pref_ou = $args{pref_ou};
- $log->info("pref_ou = $pref_ou");
- if (my ($filter) = $query->parse_tree->find_filter('pref_ou')) {
- $pref_ou = $filter->args->[0] if (@{$filter->args});
- }
- $pref_ou = actor::org_unit->search( { shortname => $pref_ou } )->next->id if ($pref_ou and $pref_ou !~ /^(-)?\d+$/);
-
- # get the default $ou if we have nothing
- $ou = actor::org_unit->search( { parent_ou => undef } )->next->id if (!$ou and !$lasso and !$mylasso);
-
-
- # XXX when user lassos are here, check to make sure we don't have one -- it'll be passed in the depth, with an ou of 0
- # gather the depth, if one is specified, defaulting to the in-query version
- my $depth = $args{depth};
- if (my ($filter) = $query->parse_tree->find_filter('depth')) {
- $depth = $filter->args->[0] if (@{$filter->args});
- }
- $depth = actor::org_unit->search_where( [{ name => $depth },{ opac_label => $depth }], {limit => 1} )->next->id if ($depth and $depth !~ /^\d+$/);
-
-
- # gather the limit or default to 10
- my $limit = $args{check_limit} || 'NULL';
- if (my ($filter) = $query->parse_tree->find_filter('limit')) {
- $limit = $filter->args->[0] if (@{$filter->args});
- }
- if (my ($filter) = $query->parse_tree->find_filter('check_limit')) {
- $limit = $filter->args->[0] if (@{$filter->args});
- }
-
-
- # gather the offset or default to 0
- my $offset = $args{skip_check} || $args{offset} || 0;
- if (my ($filter) = $query->parse_tree->find_filter('offset')) {
- $offset = $filter->args->[0] if (@{$filter->args});
- }
- if (my ($filter) = $query->parse_tree->find_filter('skip_check')) {
- $offset = $filter->args->[0] if (@{$filter->args});
- }
-
-
- # gather the estimation strategy or default to inclusion
- my $estimation_strategy = $args{estimation_strategy} || 'inclusion';
- if (my ($filter) = $query->parse_tree->find_filter('estimation_strategy')) {
- $estimation_strategy = $filter->args->[0] if (@{$filter->args});
- }
-
-
- # gather the estimation strategy or default to inclusion
- my $core_limit = $args{core_limit};
- if (my ($filter) = $query->parse_tree->find_filter('core_limit')) {
- $core_limit = $filter->args->[0] if (@{$filter->args});
- }
-
-
- # gather statuses, and then forget those if we have an #available modifier
- my @statuses;
- if (my ($filter) = $query->parse_tree->find_filter('statuses')) {
- @statuses = @{$filter->args} if (@{$filter->args});
- }
- @statuses = (0,7,12) if ($query->parse_tree->find_modifier('available'));
-
-
- # gather locations
- my @location;
- if (my ($filter) = $query->parse_tree->find_filter('locations')) {
- @location = @{$filter->args} if (@{$filter->args});
- }
-
- # gather location_groups
- if (my ($filter) = $query->parse_tree->find_filter('location_groups')) {
- my @loc_groups = @{$filter->args} if (@{$filter->args});
-
- # collect the mapped locations and add them to the locations() filter
- if (@loc_groups) {
-
- my $cstore = OpenSRF::AppSession->create( 'open-ils.cstore' );
- my $maps = $cstore->request(
- 'open-ils.cstore.direct.asset.copy_location_group_map.search.atomic',
- {lgroup => \@loc_groups})->gather(1);
-
- push(@location, $_->location) for @$maps;
- }
- }
-
-
- my $param_check = $limit || $query->superpage_size || 'NULL';
- my $param_offset = $offset || 'NULL';
- my $param_limit = $core_limit || 'NULL';
-
- my $sp = $query->superpage || 1;
- if ($sp > 1) {
- $param_offset = ($sp - 1) * $sp_size;
- }
-
- my $param_search_ou = $ou;
- my $param_depth = $depth; $param_depth = 'NULL' unless (defined($depth) and length($depth) > 0 );
- my $param_core_query = "\$core_query_$$\$" . $query->parse_tree->toSQL . "\$core_query_$$\$";
- my $param_statuses = '$${' . join(',', map { s/\$//go; "\"$_\""} @statuses) . '}$$';
- my $param_locations = '$${' . join(',', map { s/\$//go; "\"$_\""} @location) . '}$$';
- my $staff = ($self->api_name =~ /staff/ or $query->parse_tree->find_modifier('staff')) ? "'t'" : "'f'";
- my $metarecord = ($self->api_name =~ /metabib/ or $query->parse_tree->find_modifier('metabib') or $query->parse_tree->find_modifier('metarecord')) ? "'t'" : "'f'";
- my $param_pref_ou = $pref_ou || 'NULL';
-
- my $sth = metabib::metarecord_source_map->db_Main->prepare(<<" SQL");
- SELECT * -- bib search: $args{query}
- FROM search.query_parser_fts(
- $param_search_ou\:\:INT,
- $param_depth\:\:INT,
- $param_core_query\:\:TEXT,
- $param_statuses\:\:INT[],
- $param_locations\:\:INT[],
- $param_offset\:\:INT,
- $param_check\:\:INT,
- $param_limit\:\:INT,
- $metarecord\:\:BOOL,
- $staff\:\:BOOL,
- $param_pref_ou\:\:INT
- );
- SQL
-
+ my $sth = metabib::metarecord_source_map->db_Main->prepare($query->parse_tree->toSQL);
$sth->execute;
my $recs = $sth->fetchall_arrayref({});
- my $summary_row = pop @$recs;
+ $log->debug("Search yielded ".scalar(@$recs)." checked, visible results.",DEBUG);
- my $total = $$summary_row{total};
- my $checked = $$summary_row{checked};
- my $visible = $$summary_row{visible};
- my $deleted = $$summary_row{deleted};
- my $excluded = $$summary_row{excluded};
-
- my $estimate = $visible;
- if ( $total > $checked && $checked ) {
-
- $$summary_row{hit_estimate} = FTS_paging_estimate($self, $client, $checked, $visible, $excluded, $deleted, $total);
- $estimate = $$summary_row{estimated_hit_count} = $$summary_row{hit_estimate}{$estimation_strategy};
-
- }
-
- delete $$summary_row{id};
- delete $$summary_row{rel};
- delete $$summary_row{record};
-
- if (defined($simple_plan)) {
- $$summary_row{complex_query} = $simple_plan ? 0 : 1;
- } else {
- $$summary_row{complex_query} = $query->simple_plan ? 0 : 1;
- }
-
- $client->respond( $summary_row );
-
- $log->debug("Search yielded ".scalar(@$recs)." checked, visible results with an approximate visible total of $estimate.",DEBUG);
+ $client->respond({visible => scalar(@$recs)});
for my $rec (@$recs) {
- delete $$rec{checked};
- delete $$rec{visible};
- delete $$rec{excluded};
- delete $$rec{deleted};
- delete $$rec{total};
$$rec{rel} = sprintf('%0.3f',$$rec{rel});
$client->respond( $rec );
commit c7c3d1bcfd7e394f5698ea0615ad126d71741693
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Fri Sep 7 14:13:08 2012 -0400
Queryparser Driver: SQL Generation Tweaks
Remove fwhere/where distinction due to issues with detecting where some
operators were supposed to go.
Change format to a callback instead of forcing it to the top of the tree.
Change date-based filters to work in nested situations.
Change container and record_list to work in nested situations.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index 4d16d88..1b95080 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -7,6 +7,7 @@ use base 'QueryParser';
use OpenSRF::Utils::JSON;
use OpenILS::Application::AppUtils;
use OpenILS::Utils::CStoreEditor;
+use Switch;
my $U = 'OpenILS::Application::AppUtils';
my ${spc} = ' ' x 2;
@@ -44,6 +45,21 @@ sub filter_group_entry_callback {
);
}
+sub format_callback {
+ my ($invocant, $self, $struct, $filter, $params, $negate) = @_;
+
+ my $return = '';
+ my $negate_flag = ($negate ? '-' : '');
+ if(@$params[0]) {
+ my ($t,$f) = split('-', @$params[0]);
+ $return .= $negate_flag .'item_type(' . join(',',split('', $t)) . ')' if ($t);
+ $return .= ' ' if ($t and $f);
+ $return .= $negate_flag .'item_form(' . join(',',split('', $f)) . ')' if ($f);
+ $return = '(' . $return . ')' if ($t and $f);
+ }
+ return $return;
+}
+
sub quote_value {
my $self = shift;
my $value = shift;
@@ -453,7 +469,7 @@ __PACKAGE__->add_search_filter( 'saved_query', sub { return __PACKAGE__->subquer
__PACKAGE__->add_search_filter( 'filter_group_entry', sub { return __PACKAGE__->filter_group_entry_callback(@_) } );
# will be retained simply for back-compat
-__PACKAGE__->add_search_filter( 'format' );
+__PACKAGE__->add_search_filter( 'format', sub { return __PACKAGE__->format_callback(@_) } );
# grumble grumble, special cases against date1 and date2
__PACKAGE__->add_search_filter( 'before' );
@@ -518,12 +534,6 @@ sub toSQL {
my $self = shift;
my %filters;
- my ($format) = $self->find_filter('format');
- if ($format) {
- my ($t,$f) = split('-', $format->args->[0]);
- $self->new_filter( item_type => [ split '', $t ] ) if ($t);
- $self->new_filter( item_form => [ split '', $f ] ) if ($f);
- }
for my $f ( qw/preferred_language preferred_language_multiplier preferred_language_weight core_limit check_limit skip_check superpage superpage_size/ ) {
my $col = $f;
@@ -560,9 +570,7 @@ sub toSQL {
}
$rel = "1.0/($rel)::NUMERIC";
- my $mra_join = 'INNER JOIN metabib.record_attr mrd ON (m.source = mrd.id';
- $mra_join .= ' AND '. $flat_plan->{fwhere} if $flat_plan->{fwhere};
- $mra_join .= ')';
+ my $mra_join = 'INNER JOIN metabib.record_attr mrd ON m.source = mrd.id';
my $rank = $rel;
@@ -586,83 +594,6 @@ sub toSQL {
my $key = 'm.source';
$key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->modifiers});
- my ($before) = $self->find_filter('before');
- my ($after) = $self->find_filter('after');
- my ($during) = $self->find_filter('during');
- my ($between) = $self->find_filter('between');
- my ($container) = $self->find_filter('container');
- my ($record_list) = $self->find_filter('record_list');
-
- if ($before and @{$before->args} == 1) {
- $before = "AND (mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($before->args->[0]);
- } else {
- $before = '';
- }
-
- if ($after and @{$after->args} == 1) {
- $after = "AND (mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($after->args->[0]);
- } else {
- $after = '';
- }
-
- if ($during and @{$during->args} == 1) {
- $during = "AND " . $self->QueryParser->quote_value($during->args->[0]) . " BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2')";
- } else {
- $during = '';
- }
-
- if ($between and @{$between->args} == 2) {
- $between = "AND (mrd.attrs->'date1') BETWEEN " . $self->QueryParser->quote_value($between->args->[0]) . " AND " . $self->QueryParser->quote_value($between->args->[1]);
- } else {
- $between = '';
- }
-
- if ($container and @{$container->args} >= 3) {
- my ($class, $ctype, $cid, $token) = @{ $container->args };
-
- my $perm_join = '';
- my $rec_join = '';
- my $rec_field = 'ci.target_biblio_record_entry';
-
- if ($class eq 'bre') {
- $class = 'biblio_record_entry';
- } elsif ($class eq 'acn') {
- $class = 'call_number';
- $rec_field = 'cn.record';
- $rec_join = 'JOIN asset.call_number cn ON (ci.target_call_number = cn.id)';
- } elsif ($class eq 'acp') {
- $class = 'copy';
- $rec_field = 'cn.record';
- $rec_join = 'JOIN asset.copy cp ON (ci.target_copy = cp.id) JOIN asset.call_number cn ON (cp.call_number = cn.id)';
-# } elsif ($class eq 'au') {
-# $class = 'user';
-# $rec_field = 'cn.record';
-# $rec_join = 'JOIN asset.call_number cn ON ci.target_call_number = cn.id';
- } else { $class = undef };
-
- if ($class) {
- my ($u,$e) = $apputils->checksesperm($token) if ($token);
- $perm_join = 'OR c.owner = ' . $u->id if ($u && !$e);
-
- $container = qq<
- JOIN ( SELECT $rec_field AS container_item
- FROM container.${class}_bucket_item ci
- JOIN container.${class}_bucket c ON (c.id = ci.bucket)
- $rec_join
- WHERE c.btype = > . $self->QueryParser->quote_value($ctype) .
- qq< AND c.id = > . $self->QueryParser->quote_value($cid) .
- qq< AND (c.pub IS TRUE $perm_join)) container ON (container.container_item = m.source) >;
- } else {$container = ''};
- } else {
- $container = '';
- }
-
- if ($record_list and @{$record_list->args} > 0) {
- $record_list = 'JOIN (VALUES (' . join('),(', map { $self->QueryParser->quote_value($_) } @{ $record_list->args}) . ")) record_list(id) ON (record_list.id::BIGINT = $key)"
- } else {
- $record_list = '';
- }
-
my $core_limit = $self->QueryParser->core_limit || 25000;
my $flat_where = $$flat_plan{where};
@@ -690,15 +621,9 @@ SELECT $key AS id,
$rank AS rank,
FIRST(mrd.attrs->'date1') AS tie_break
FROM metabib.metarecord_source_map m
- $container
- $record_list
$$flat_plan{from}
$mra_join
WHERE 1=1
- $before
- $after
- $during
- $between
$flat_where
GROUP BY 1
ORDER BY 4 $desc $nullpos, 5 DESC $nullpos, 3 DESC
@@ -739,14 +664,6 @@ sub flatten {
my $from = shift || '';
my $where = shift || '(';
my $with = '';
- my $fwhere = shift || ''; # for joining dynamic filters (mra)
-
- my @dyn_filters;
- for my $filter (@{$self->filters}) {
- push(@dyn_filters, $filter) if
- grep { $_ eq $filter->name }
- @{ $self->QueryParser->dynamic_filters };
- };
my @rank_list;
for my $node ( @{$self->query_nodes} ) {
@@ -817,18 +734,10 @@ sub flatten {
}
- my $twhere .= '(' . $talias . ".id IS NOT NULL";
- $twhere .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
- $twhere .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
- $twhere .= ')';
-
- if (@dyn_filters or !$self->top_plan) {
- # if this WHERE is represented within the dynamic
- # filter's ON clause, it's not also needed in the main WHERE.
- $fwhere .= $twhere;
- } else {
- $where .= $twhere;
- }
+ $where .= '(' . $talias . ".id IS NOT NULL";
+ $where .= ' AND ' . join(' AND ', map {"${talias}.value ~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->phrases}) if (@{$node->phrases});
+ $where .= ' AND ' . join(' AND ', map {"${talias}.value !~* ".$self->QueryParser->quote_phrase_value($_)} @{$node->unphrases}) if (@{$node->unphrases});
+ $where .= ')';
push @rank_list, $node_rank;
@@ -844,35 +753,29 @@ sub flatten {
@field_ids = @{ $self->QueryParser->facet_field_ids_by_class( $node->classname ) };
}
- my $join_type = ($node->negate or @dyn_filters or !$self->top_plan) ? 'LEFT' : 'INNER';
+ my $join_type = ($node->negate or !$self->top_plan) ? 'LEFT' : 'INNER';
$from .= "\n${spc}$join_type JOIN /* facet */ metabib.facet_entry $talias ON (\n${spc}${spc}m.source = ${talias}.source\n${spc}${spc}".
"AND SUBSTRING(${talias}.value,1,1024) IN (" . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n${spc}${spc}".
"AND ${talias}.field IN (". join(',', @field_ids) . ")\n${spc})";
- if (@dyn_filters or !$self->top_plan) {
+ if ($join_type != 'INNER') {
my $NOT = $node->negate ? '' : ' NOT';
- $fwhere .= "${talias}.id IS$NOT NULL";
- } else {
- $where .= $node->negate ? "${talias}.id IS NULL" : 'TRUE';
+ $where .= "${talias}.id IS$NOT NULL";
}
} else {
my $subnode = $node->flatten;
# strip the trailing bool from the previous loop if there is
- # nothing to add to the where/fwhere within this loop.
+ # nothing to add to the where within this loop.
if ($$subnode{where} eq '()') {
$where =~ s/\s(AND|OR)\s$//;
-
- } elsif ($$subnode{fwhere} eq '') {
- $fwhere =~ s/\s(AND|OR)\s$//;
}
push(@rank_list, @{$$subnode{rank_list}});
$from .= $$subnode{from};
- $where .= "($$subnode{where})" unless $$subnode{where} eq '()';
- $fwhere .= "($$subnode{fwhere})" if $$subnode{fwhere};
+ $where .= "$$subnode{where}" unless $$subnode{where} eq '()';
if ($$subnode{with}) {
$with .= ",\n " if $with;
@@ -883,44 +786,111 @@ sub flatten {
warn "flatten(): appending WHERE bool to: $where\n" if $self->QueryParser->debug;
- if ($fwhere) {
- # bool joiner for inter-plan filters
- $fwhere .= ' AND ' if ($node eq '&');
- $fwhere .= ' OR ' if ($node eq '|');
-
- } elsif ($where ne '(') {
-
+ if ($where ne '(') {
$where .= ' AND ' if ($node eq '&');
$where .= ' OR ' if ($node eq '|');
}
}
}
- # for each dynamic filter, build the ON clause for the JOIN
- for my $filter (@dyn_filters) {
-
- warn "flatten(): processing dynamic filter ". $filter->name ."\n"
- if $self->QueryParser->debug;
+ my $joiner = sprintf(" %s ", ($self->joiner eq '&' ? 'AND' : 'OR'));
+ # for each dynamic filter, build more of the WHERE clause
+ for my $filter (@{$self->filters}) {
+ if (grep { $_ eq $filter->name } @{ $self->QueryParser->dynamic_filters }) {
+
+ warn "flatten(): processing dynamic filter ". $filter->name ."\n"
+ if $self->QueryParser->debug;
- # bool joiner for intra-plan nodes/filters
- $fwhere .= sprintf(" %s ", ($self->joiner eq '&' ? 'AND' : 'OR')) if $fwhere;
+ # bool joiner for intra-plan nodes/filters
+ $where .= $joiner if $where ne '(';
- my @fargs = @{$filter->args};
- my $NOT = $filter->negate ? ' NOT' : '';
- my $fname = $filter->name;
- $fname = 'item_lang' if $fname eq 'language'; #XXX filter aliases
+ my @fargs = @{$filter->args};
+ my $NOT = $filter->negate ? ' NOT' : '';
+ my $fname = $filter->name;
+ $fname = 'item_lang' if $fname eq 'language'; #XXX filter aliases
- $fwhere .= sprintf(
- "attrs->'%s'$NOT IN (%s)", $fname,
- join(',', map { $self->QueryParser->quote_value($_) } @fargs)
- );
+ $where .= sprintf(
+ "attrs->'%s'$NOT IN (%s)", $fname,
+ join(',', map { $self->QueryParser->quote_value($_) } @fargs)
+ );
- warn "flatten(): filter where => $fwhere\n"
- if $self->QueryParser->debug;
+ warn "flatten(): filter where => $where\n"
+ if $self->QueryParser->debug;
+ } else {
+ my $NOT = $filter->negate ? 'NOT ' : '';
+ switch ($filter->name) {
+ case 'before' {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= "$NOT(mrd.attrs->'date1') <= " . $self->QueryParser->quote_value($filter->args->[0]);
+ }
+ }
+ case 'after' {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= "$NOT(mrd.attrs->'date1') >= " . $self->QueryParser->quote_value($filter->args->[0]);
+ }
+ }
+ case 'during' {
+ if (@{$filter->args} == 1) {
+ $where .= $joiner if $where ne '(';
+ $where .= $self->QueryParser->quote_value($filter->args->[0]) . " ${NOT}BETWEEN (mrd.attrs->'date1') AND (mrd.attrs->'date2')";
+ }
+ }
+ case 'between' {
+ if (@{$filter->args} == 2) {
+ $where .= $joiner if $where ne '(';
+ $where .= "(mrd.attrs->'date1') ${NOT}BETWEEN " . $self->QueryParser->quote_value($filter->args->[0]) . " AND " . $self->QueryParser->quote_value($filter->args->[1]);
+ }
+ }
+ case 'container' {
+ if (@{$filter->args} >= 3) {
+ my ($class, $ctype, $cid, $token) = @{$filter->args};
+ my $perm_join = '';
+ my $rec_join = '';
+ my $rec_field = 'ci.target_biblio_record_entry';
+ switch($class) {
+ case 'bre' {
+ $class = 'biblio_record_entry';
+ }
+ case 'acn' {
+ $class = 'call_number';
+ $rec_field = 'cn.record';
+ $rec_join = 'JOIN asset.call_number cn ON (ci.target_call_number = cn.id)';
+ }
+ case 'acp' {
+ $class = 'copy';
+ $rec_field = 'cn.record';
+ $rec_join = 'JOIN asset.copy cp ON (ci.target_copy = cp.id) JOIN asset.call_number cn ON (cp.call_number = cn.id)';
+ }
+ else {
+ $class = undef;
+ }
+ }
+
+ if ($class) {
+ my ($u,$e) = $apputils->checksesperm($token) if ($token);
+ $perm_join = ' OR c.owner = ' . $u->id if ($u && !$e);
+ $where .= $joiner if $where ne '(';
+ $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)"
+ }
+ }
+ }
+ case 'record_list' {
+ if (@{$filter->args} > 0) {
+ my $key = 'm.source';
+ $key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers});
+ $where .= $joiner if $where ne '(';
+ $where .= 'NOT ' if $filter->negate;
+ $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args}) . ')';
+ }
+ }
+ }
+ }
}
- warn "flatten(): full filter where => $fwhere\n" if $self->QueryParser->debug;
+ warn "flatten(): full filter where => $where\n" if $self->QueryParser->debug;
- return { rank_list => \@rank_list, from => $from, where => $where.')', with => $with, fwhere => $fwhere };
+ return { rank_list => \@rank_list, from => $from, where => $where.')', with => $with };
}
commit 7e2dd736ffe0dbc969ce4e365efe8834889a103a
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Fri Sep 7 10:46:34 2012 -0400
QueryParser Driver: Adjust query whitespace
Adjust spacing and newlines to make the resulting query more easily read.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index d566a37..4d16d88 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -543,7 +543,7 @@ sub toSQL {
my $flat_plan = $self->flatten;
# generate the relevance ranking
- my $rel = "AVG(\n${spc}${spc}(" . join(")+\n${spc}${spc}(", @{$$flat_plan{rank_list}}) . ")\n${spc})+1";
+ my $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1";
# find any supplied sort option
my ($sort_filter) = $self->find_filter('sort');
@@ -721,12 +721,12 @@ sub rel_bump {
return '' if (!@$only_atoms);
if ($bump eq 'first_word') {
- return " /* first_word */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||search_normalize(".$self->QueryParser->quote_phrase_value($only_atoms->[0]->content)."))), FALSE )::INT * $multiplier, 1)";
+ return "/* first_word */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||search_normalize(".$self->QueryParser->quote_phrase_value($only_atoms->[0]->content)."))), FALSE )::INT * $multiplier, 1)";
} elsif ($bump eq 'full_match') {
- return " /* full_match */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||".
+ return "/* full_match */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ ('^'||".
join( "||' '||", map { "search_normalize(".$self->QueryParser->quote_phrase_value($_->content).")" } @$only_atoms )."||'\$')), FALSE )::INT * $multiplier, 1)";
} elsif ($bump eq 'word_order') {
- return " /* word_order */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ (".
+ return "/* word_order */ COALESCE(NULLIF( (search_normalize(".$node->table_alias.".value) ~ (".
join( "||'.*'||", map { "search_normalize(".$self->QueryParser->quote_phrase_value($_->content).")" } @$only_atoms ).")), FALSE )::INT * $multiplier, 1)";
}
@@ -766,15 +766,15 @@ sub flatten {
my $node_rank = 'COALESCE(' . $node->rank . " * ${talias}.weight, 0.0)";
my $core_limit = $self->QueryParser->core_limit || 25000;
- $from .= "\n${spc}LEFT JOIN (\n${spc}${spc}SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq /* search */\n${spc}${spc} FROM $table AS fe";
- $from .= "\n${spc}${spc}${spc}JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
+ $from .= "\n${spc}${spc}${spc}${spc}LEFT JOIN (\n${spc}${spc}${spc}${spc}${spc}SELECT fe.*, fe_weight.weight, ${talias}_xq.tsq /* search */\n${spc}${spc}${spc}${spc}${spc} FROM $table AS fe";
+ $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}JOIN config.metabib_field AS fe_weight ON (fe_weight.id = fe.field)";
if ($node->dummy_count < @{$node->only_atoms} ) {
- $with .= ",\n" if $with;
+ $with .= ",\n " if $with;
$with .= "${talias}_xq AS (SELECT ". $node->tsquery ." AS tsq )";
- $from .= "\n${spc}${spc}${spc}JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
+ $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}JOIN ${talias}_xq ON (fe.index_vector @@ ${talias}_xq.tsq)";
} else {
- $from .= "\n${spc}${spc}${spc}, (SELECT NULL::tsquery AS tsq ) AS x";
+ $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}, (SELECT NULL::tsquery AS tsq ) AS x";
}
my @bump_fields;
@@ -789,7 +789,7 @@ sub flatten {
} @bump_fields
);
if (@field_ids) {
- $from .= "\n${spc}${spc}${spc}WHERE fe_weight.id IN (" .
+ $from .= "\n${spc}${spc}${spc}${spc}${spc}${spc}WHERE fe_weight.id IN (" .
join(',', @field_ids) . ")";
}
@@ -798,7 +798,7 @@ sub flatten {
}
###$from .= "\n${spc}${spc}LIMIT $core_limit";
- $from .= "\n${spc}) AS $talias ON (m.source = ${talias}.source)";
+ $from .= "\n${spc}${spc}${spc}${spc}) AS $talias ON (m.source = ${talias}.source)";
my %used_bumps;
@@ -812,7 +812,7 @@ sub flatten {
next if ($$bumps{$b}{multiplier} == 1); # optimization to remove unneeded bumps
my $bump_case = $self->rel_bump( $node, $b, $$bumps{$b}{multiplier} );
- $node_rank .= "\n${spc}${spc}${spc}${spc} * " . $bump_case if ($bump_case);
+ $node_rank .= "\n${spc}${spc}${spc}${spc}${spc}* " . $bump_case if ($bump_case);
}
}
@@ -875,8 +875,8 @@ sub flatten {
$fwhere .= "($$subnode{fwhere})" if $$subnode{fwhere};
if ($$subnode{with}) {
- $with .= ', ' if $with;
- $with .= " " . $$subnode{with};
+ $with .= ",\n " if $with;
+ $with .= $$subnode{with};
}
}
} else {
commit 9626889ea9c693b2576593591417dcbf11306f93
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Sun Sep 16 16:46:43 2012 -0400
QueryParser: Expand negate and disallow operators
Allow negate to act like disallow on phrases.
Allow both to apply to groups.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 0661b5b..02deeeb 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -890,6 +890,9 @@ sub decompose {
my $disallowed_op = $pkg->operator('disallowed');
my $disallowed_re = qr/\Q$disallowed_op\E/;
+ my $negated_op = $pkg->operator('negated');
+ my $negated_re = qr/\Q$negated_op\E/;
+
my $and_op = $pkg->operator('and');
my $and_re = qr/^\s*\Q$and_op\E/;
@@ -897,7 +900,7 @@ sub decompose {
my $or_re = qr/^\s*\Q$or_op\E/;
my $group_start = $pkg->operator('group_start');
- my $group_start_re = qr/^\s*\Q$group_start\E/;
+ my $group_start_re = qr/^\s*($negated_re|$disallowed_re)?\Q$group_start\E/;
my $group_end = $pkg->operator('group_end');
my $group_end_re = qr/^\s*\Q$group_end\E/;
@@ -911,9 +914,6 @@ sub decompose {
my $modifier_tag = $pkg->operator('modifier');
my $modifier_tag_re = qr/^\s*\Q$modifier_tag\E/;
- my $negated_op = $pkg->operator('negated');
- my $negated_re = qr/\Q$negated_op\E/;
-
# Group start/end normally are ( and ), but can be overridden.
# We thus include ( and ) specifically due to filters, as well as : for classes.
my $phrase_cleanup_re = qr/\s*(\Q$required_op\E|\Q$disallowed_op\E|\Q$and_op\E|\Q$or_op\E|\Q$group_start\E|\Q$group_end\E|\Q$float_start\E|\Q$float_end\E|\Q$modifier_tag\E|\Q$negated_op\E|:|\(|\))/;
@@ -1041,8 +1041,9 @@ sub decompose {
$last_type = '';
} elsif (/$group_start_re/) { # start of an explicit group
warn ' 'x$recursing."Encountered explicit group start\n" if $self->debug;
-
+ my $negate = $1;
my ($substruct, $subremainder) = $self->decompose( $', $current_class, $recursing + 1 );
+ $substruct->negate(1) if ($substruct && $negate);
$struct->add_node( $substruct ) if ($substruct);
$_ = $subremainder;
warn ' 'x$recursing."Query remainder after bool group: $_\n" if $self->debug;
@@ -1140,10 +1141,11 @@ sub decompose {
$_ = $';
local $last_type = 'CLASS';
- } elsif (/^\s*($required_re|$disallowed_re)?"([^"]+)"/) { # phrase, always anded
+ } elsif (/^\s*($required_re|$disallowed_re|$negated_re)?"([^"]+)"/) { # phrase, always anded
warn ' 'x$recursing.'Encountered' . ($1 ? " ['$1' modified]" : '') . " phrase: $2\n" if $self->debug;
my $req_ness = $1 || '';
+ $req_ness = $disallowed_op if ($req_ness eq $negated_op);
my $phrase = $2;
if (!$phrase_helper) {
@@ -1163,10 +1165,13 @@ sub decompose {
}
$class_node->add_phrase( $phrase );
+ # Save $' before we clean up $phrase
+ my $temp_val = $';
+
# Cleanup the phrase to make it so that we don't parse things in it as anything other than atoms
$phrase =~ s/$phrase_cleanup_re/ /g;
- $_ = $phrase . $';
+ $_ = $phrase . $temp_val;
}
@@ -1388,8 +1393,10 @@ sub abstract_query2str_impl {
my $ge = $qpconfig->{operators}{group_end};
my $and = $qpconfig->{operators}{and};
my $or = $qpconfig->{operators}{or};
+ my $ng = $qpconfig->{operators}{negated};
my $isnode = 0;
+ my $negate = '';
my $size = 0;
my $q = "";
@@ -1402,6 +1409,10 @@ sub abstract_query2str_impl {
exists $abstract_query->{modifiers};
$size = _kid_list($abstract_query->{children});
+ if ($abstract_query->{negate}) {
+ $isnode = 1;
+ $negate = $ng;
+ }
$isnode = 1 if ($size > 1 and ($force_qp_node or $depth));
#warn "size: $size, depth: $depth, isnode: $isnode, AQ: ".Dumper($abstract_query);
} elsif ($abstract_query->{type} eq 'node') {
@@ -1463,6 +1474,7 @@ sub abstract_query2str_impl {
}
$q = "$gs$q$ge" if ($isnode);
+ $q = $negate . $q if ($q);;
return $q;
}
@@ -1747,6 +1759,15 @@ sub add_filter {
return $self;
}
+sub negate {
+ my $self = shift;
+ my $negate = shift;
+
+ $self->{negate} = $negate if (defined $negate);
+
+ return $self->{negate};
+}
+
# %opts supports two options at this time:
# no_phrases :
# If true, do not do anything to the phrases
@@ -1765,7 +1786,8 @@ sub to_abstract_query {
floating => $self->floating,
level => $self->plan_level,
filters => [map { $_->to_abstract_query } @{$self->filters}],
- modifiers => [map { $_->to_abstract_query } @{$self->modifiers}]
+ modifiers => [map { $_->to_abstract_query } @{$self->modifiers}],
+ negate => $self->negate
};
if ($opts{with_config}) {
commit 1199e3835f8308ca5a9d9b5329a60594e4710ef5
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Sun Sep 16 15:21:07 2012 -0400
QueryParser: Add negate operator
To replace the previous -atom behavior add a negate op, by default !.
This acts identically to how - worked before it was changed to convert atoms
into unphrases.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 3c875cb..0661b5b 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -35,7 +35,8 @@ our %parser_config = (
group_end => ')',
required => '+',
disallowed => '-',
- modifier => '#'
+ modifier => '#',
+ negated => '!'
}
}
);
@@ -910,9 +911,12 @@ sub decompose {
my $modifier_tag = $pkg->operator('modifier');
my $modifier_tag_re = qr/^\s*\Q$modifier_tag\E/;
+ my $negated_op = $pkg->operator('negated');
+ my $negated_re = qr/\Q$negated_op\E/;
+
# Group start/end normally are ( and ), but can be overridden.
# We thus include ( and ) specifically due to filters, as well as : for classes.
- my $phrase_cleanup_re = qr/\s*(\Q$required_op\E|\Q$disallowed_op\E|\Q$and_op\E|\Q$or_op\E|\Q$group_start\E|\Q$group_end\E|\Q$float_start\E|\Q$float_end\E|\Q$modifier_tag\E|:|\(|\))/;
+ my $phrase_cleanup_re = qr/\s*(\Q$required_op\E|\Q$disallowed_op\E|\Q$and_op\E|\Q$or_op\E|\Q$group_start\E|\Q$group_end\E|\Q$float_start\E|\Q$float_end\E|\Q$modifier_tag\E|\Q$negated_op\E|:|\(|\))/;
# Build the filter and modifier uber-regexps
my $facet_re = '^\s*(-?)((?:' . join( '|', @{$pkg->facet_classes}) . ')(?:\|\w+)*)\[(.+?)\]';
@@ -1186,7 +1190,7 @@ sub decompose {
my $class_node = $struct->classed_node($current_class);
- my $prefix = ($atom =~ s/^$disallowed_re//o) ? '!' : '';
+ my $prefix = ($atom =~ s/^$negated_re//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
@@ -1412,7 +1416,7 @@ sub abstract_query2str_impl {
$isnode = 1;
} elsif ($abstract_query->{type} eq 'atom') {
my $prefix = $abstract_query->{prefix} || '';
- $prefix = $qpconfig->{operators}{disallowed} if $prefix eq '!';
+ $prefix = $qpconfig->{operators}{negated} if $prefix eq '!';
$q .= ($q ? ' ' : '') . $prefix .
($abstract_query->{content} || '') .
($abstract_query->{suffix} || '');
commit 9c2df12c20f73619f427fca20fa4b79e167df35c
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Sun Sep 16 15:08:06 2012 -0400
QueryParser: Treat Unphrases as negated phrases
This should allow drivers to better check for negated phrases by using
indexes on the contained atoms.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 28f2c70..3c875cb 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -1154,14 +1154,10 @@ sub decompose {
my $class_node = $struct->classed_node($current_class);
- if ($req_ness eq $pkg->operator('disallowed')) {
- $class_node->add_dummy_atom( node => $class_node );
- $class_node->add_unphrase( $phrase );
- $phrase = '';
- #$phrase =~ s/(^|\s)\b/$1-/g;
- } else {
- $class_node->add_phrase( $phrase );
+ if ($req_ness eq $disallowed_op) {
+ $class_node->negate(1);
}
+ $class_node->add_phrase( $phrase );
# Cleanup the phrase to make it so that we don't parse things in it as anything other than atoms
$phrase =~ s/$phrase_cleanup_re/ /g;
@@ -1195,7 +1191,6 @@ sub decompose {
if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
# $class_node->add_phrase( $atom ) if ($atom =~ s/^$required_re//o);
-# $class_node->add_unphrase( $atom ) if ($prefix eq '!');
$class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $prefix, node => $class_node );
$struct->joiner( '&' );
@@ -1750,7 +1745,7 @@ sub add_filter {
# %opts supports two options at this time:
# no_phrases :
-# If true, do not do anything to the phrases and unphrases
+# If true, do not do anything to the phrases
# fields on any discovered nodes.
# with_config :
# If true, also return the query parser config as part of the blob.
@@ -1899,15 +1894,6 @@ sub phrases {
return $self->{phrases};
}
-sub unphrases {
- my $self = shift;
- my @phrases = @_;
-
- $self->{unphrases} ||= [];
- $self->{unphrases} = \@phrases if (@phrases);
- return $self->{unphrases};
-}
-
sub add_phrase {
my $self = shift;
my $phrase = shift;
@@ -1917,13 +1903,13 @@ sub add_phrase {
return $self;
}
-sub add_unphrase {
+sub negate {
my $self = shift;
- my $phrase = shift;
+ my $negate = shift;
- push(@{$self->unphrases}, $phrase);
+ $self->{negate} = $negate if (defined $negate);
- return $self;
+ return $self->{negate};
}
sub query_atoms {
@@ -2037,7 +2023,10 @@ sub to_abstract_query {
# break them into atoms as QP would, and remove any matching
# sequences of atoms from our abstract query.
- my $tmptree = $self->{plan}->{QueryParser}->new(query => '"'.$phrase.'"')->parse->parse_tree;
+ my $tmp_prefix = '';
+ $tmp_prefix = $QueryParser::parser_config{$pkg}{operators}{disallowed} if ($self->{negate});
+
+ my $tmptree = $self->{plan}->{QueryParser}->new(query => $tmp_prefix.'"'.$phrase.'"')->parse->parse_tree;
if ($tmptree) {
# For a well-behaved phrase, we should now have only one node
# in the $tmptree query plan, and that node should have an
@@ -2059,40 +2048,7 @@ sub to_abstract_query {
last if $self->replace_phrase_in_abstract_query(
$tmplist,
$_,
- QueryParser::_util::fake_abstract_atom_from_phrase($phrase, undef, $pkg)
- );
- }
- }
- }
- }
- }
-
- # Do the same as the preceding block for unphrases (negated phrases).
- if ($self->{unphrases} and not $opts{no_phrases}) {
- for my $phrase (@{$self->{unphrases}}) {
- my $tmptree = $self->{plan}->{QueryParser}->new(
- query => $QueryParser::parser_config{$pkg}{operators}{disallowed}.
- '"' . $phrase . '"'
- )->parse->parse_tree;
-
- if ($tmptree) {
- if ($tmptree->{query} and scalar(@{$tmptree->{query}}) == 1) {
- my $tmplist;
-
- eval {
- $tmplist = $tmptree->{query}->[0]->to_abstract_query(
- no_phrases => 1
- )->{children}->{'&'}->[0]->{children}->{'&'};
- };
- next if $@;
-
- foreach (
- QueryParser::_util::find_arrays_in_abstract($abstract_query->{children})
- ) {
- last if $self->replace_phrase_in_abstract_query(
- $tmplist,
- $_,
- QueryParser::_util::fake_abstract_atom_from_phrase($phrase, 1, $pkg)
+ QueryParser::_util::fake_abstract_atom_from_phrase($phrase, $self->{negate}, $pkg)
);
}
}
commit 56d46e45f58616ab831247f7c6858de55e35962e
Author: Thomas Berezansky <tsbere at mvlc.org>
Date: Sun Sep 16 13:53:16 2012 -0400
QueryParser: Protect phrase parsing
Ensure that phrases don't get parsed as containing anything other than
individual atoms. This ensures that you can phrase-escape things that would
otherwise be treated as QP syntax.
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index a5950af..28f2c70 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -883,17 +883,17 @@ sub decompose {
warn ' 'x$recursing." ** Rewritten query: $_\n" if $self->debug;
warn ' 'x$recursing." ** Search class RE: $search_class_re\n" if $self->debug;
- my $required_re = $pkg->operator('required');
- $required_re = qr/\Q$required_re\E/;
+ my $required_op = $pkg->operator('required');
+ my $required_re = qr/\Q$required_op\E/;
- my $disallowed_re = $pkg->operator('disallowed');
- $disallowed_re = qr/\Q$disallowed_re\E/;
+ my $disallowed_op = $pkg->operator('disallowed');
+ my $disallowed_re = qr/\Q$disallowed_op\E/;
- my $and_re = $pkg->operator('and');
- $and_re = qr/^\s*\Q$and_re\E/;
+ my $and_op = $pkg->operator('and');
+ my $and_re = qr/^\s*\Q$and_op\E/;
- my $or_re = $pkg->operator('or');
- $or_re = qr/^\s*\Q$or_re\E/;
+ my $or_op = $pkg->operator('or');
+ my $or_re = qr/^\s*\Q$or_op\E/;
my $group_start = $pkg->operator('group_start');
my $group_start_re = qr/^\s*\Q$group_start\E/;
@@ -907,9 +907,12 @@ sub decompose {
my $float_end = $pkg->operator('float_end');
my $float_end_re = qr/^\s*\Q$float_end\E/;
- my $modifier_tag_re = $pkg->operator('modifier');
- $modifier_tag_re = qr/^\s*\Q$modifier_tag_re\E/;
+ my $modifier_tag = $pkg->operator('modifier');
+ my $modifier_tag_re = qr/^\s*\Q$modifier_tag\E/;
+ # Group start/end normally are ( and ), but can be overridden.
+ # We thus include ( and ) specifically due to filters, as well as : for classes.
+ my $phrase_cleanup_re = qr/\s*(\Q$required_op\E|\Q$disallowed_op\E|\Q$and_op\E|\Q$or_op\E|\Q$group_start\E|\Q$group_end\E|\Q$float_start\E|\Q$float_end\E|\Q$modifier_tag\E|:|\(|\))/;
# Build the filter and modifier uber-regexps
my $facet_re = '^\s*(-?)((?:' . join( '|', @{$pkg->facet_classes}) . ')(?:\|\w+)*)\[(.+?)\]';
@@ -1159,6 +1162,10 @@ sub decompose {
} else {
$class_node->add_phrase( $phrase );
}
+
+ # Cleanup the phrase to make it so that we don't parse things in it as anything other than atoms
+ $phrase =~ s/$phrase_cleanup_re/ /g;
+
$_ = $phrase . $';
}
commit 8a709bf9ca16355d74f1791483e23a13aea03ed6
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Sep 14 14:28:20 2012 -0400
Convert negated words to unphrases, like we do with +d words to phrases
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 0968a18..a5950af 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -1165,10 +1165,10 @@ sub decompose {
local $last_type = '';
- } elsif (/^\s*$required_re([^${group_end}${float_end}\s"]+)/) { # phrase, always anded
+ } elsif (/^\s*($required_re|$disallowed_re)([^${group_end}${float_end}\s"]+)/) { # convert require/disallow word to {un}phrase
warn ' 'x$recursing."Encountered required atom (mini phrase), transforming for phrase parse: $1\n" if $self->debug;
- $_ = '"' . $1 . '"' . $';
+ $_ = $1 . '"' . $2 . '"' . $';
local $last_type = '';
} elsif (/^\s*([^${group_end}${float_end}\s]+)/o) { # atom
commit efa0f86ee926d8f3e1068779b3e01eb0943c9a57
Author: Mike Rylander <mrylander at gmail.com>
Date: Wed Sep 12 13:12:12 2012 -0400
Lots ...
* increase debugging amount and readability
* floating sections (push-to-top)
* force plan level setting
* fix several forms of auto-pushdown breakage (explicit bool precedence support)
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 4746d78..0968a18 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -803,6 +803,9 @@ sub parse {
$self->floating_plan->add_node( $self->parse_tree );
$self->parse_tree( $self->floating_plan );
}
+
+ $self->parse_tree->plan_level(0);
+
return $self;
}
@@ -815,11 +818,15 @@ Returns the top level query plan, or the query plan from a lower level plus
the portion of the query string that needs to be processed at a higher level.
=cut
+our $last_class = '';
+our $last_type = '';
+our $floating = 0;
+our $fstart;
+
sub decompose {
my $self = shift;
my $pkg = ref($self) || $self;
- warn " ** decompose package is $pkg\n" if $self->debug;
$_ = shift;
my $current_class = shift || $self->default_search_class;
@@ -831,18 +838,20 @@ sub decompose {
my $search_class_re = '^\s*(';
my $first_class = 1;
+ warn ' 'x$recursing." ** decompose package is $pkg\n" if $self->debug;
+
my %seen_classes;
for my $class ( keys %{$pkg->search_field_aliases} ) {
- warn " *** ... Looking for search fields in $class\n" if $self->debug;
+ warn ' 'x$recursing." *** ... Looking for search fields in $class\n" if $self->debug;
for my $field ( keys %{$pkg->search_field_aliases->{$class}} ) {
- warn " *** ... Looking for aliases of $field\n" if $self->debug;
+ warn ' 'x$recursing." *** ... Looking for aliases of $field\n" if $self->debug;
for my $alias ( @{$pkg->search_field_aliases->{$class}{$field}} ) {
my $aliasr = qr/$alias/;
s/(^|\s+)$aliasr\|/$1$class\|$field#$alias\|/g;
s/(^|\s+)$aliasr[:=]/$1$class\|$field#$alias:/g;
- warn " *** Rewriting: $alias ($aliasr) as $class\|$field\n" if $self->debug;
+ warn ' 'x$recursing." *** Rewriting: $alias ($aliasr) as $class\|$field\n" if $self->debug;
}
}
@@ -858,7 +867,7 @@ sub decompose {
my $aliasr = qr/$alias/;
s/(^|[^|])\b$aliasr\|/$1$class#$alias\|/g;
s/(^|[^|])\b$aliasr[:=]/$1$class#$alias:/g;
- warn " *** Rewriting: $alias ($aliasr) as $class\n" if $self->debug;
+ warn ' 'x$recursing." *** Rewriting: $alias ($aliasr) as $class\n" if $self->debug;
}
if (!$seen_classes{$class}) {
@@ -871,8 +880,8 @@ sub decompose {
}
$search_class_re .= '):';
- warn " ** Rewritten query: $_\n" if $self->debug;
- warn " ** Search class RE: $search_class_re\n" if $self->debug;
+ warn ' 'x$recursing." ** Rewritten query: $_\n" if $self->debug;
+ warn ' 'x$recursing." ** Search class RE: $search_class_re\n" if $self->debug;
my $required_re = $pkg->operator('required');
$required_re = qr/\Q$required_re\E/;
@@ -904,7 +913,7 @@ sub decompose {
# Build the filter and modifier uber-regexps
my $facet_re = '^\s*(-?)((?:' . join( '|', @{$pkg->facet_classes}) . ')(?:\|\w+)*)\[(.+?)\]';
- warn " ** Facet RE: $facet_re\n" if $self->debug;
+ warn ' 'x$recursing." ** Facet RE: $facet_re\n" if $self->debug;
my $filter_re = '^\s*(-?)(' . join( '|', @{$pkg->filters}) . ')\(([^()]+)\)';
my $filter_as_class_re = '^\s*(-?)(' . join( '|', @{$pkg->filters}) . '):\s*(\S+)';
@@ -917,26 +926,37 @@ sub decompose {
my $remainder = '';
- my $last_type = '';
while (!$remainder) {
+ warn ' 'x$recursing."Start of the loop. last_type: $last_type, joiner: ".$struct->joiner.", struct: $struct\n" if $self->debug;
+ if ($last_type eq 'FEND' and $fstart and $fstart != $struct) { # fall back further
+ $remainder = $_;
+ last;
+ } elsif ($last_type eq 'FEND') {
+ $fstart = undef;
+ $last_type = '';
+ }
+
if (/^\s*$/) { # end of an explicit group
+ local $last_type = '';
last;
} elsif (/$float_end_re/) { # end of an explicit group
- warn "Encountered explicit float end\n" if $self->debug;
+ warn ' 'x$recursing."Encountered explicit float end, remainder: $'\n" if $self->debug;
$remainder = $';
$_ = '';
- $last_type = '';
+ $floating = 0;
+ $last_type = 'FEND';
+ last;
} elsif (/$group_end_re/) { # end of an explicit group
- warn "Encountered explicit group end\n" if $self->debug;
+ warn ' 'x$recursing."Encountered explicit group end, remainder: $'\n" if $self->debug;
- $_ = $';
- $remainder = $struct->top_plan ? '' : $';
+ $remainder = $';
+ $_ = '';
- $last_type = '';
+ local $last_type = '';
} elsif ($self->filter_count && /$filter_re/) { # found a filter
- warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
+ warn ' 'x$recursing."Encountered search filter: $1$2 set to $3\n" if $self->debug;
my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
@@ -952,9 +972,9 @@ sub decompose {
}
- $last_type = '';
+ local $last_type = '';
} elsif ($self->filter_count && /$filter_as_class_re/) { # found a filter
- warn "Encountered search filter: $1$2 set to $3\n" if $self->debug;
+ warn ' 'x$recursing."Encountered search filter: $1$2 set to $3\n" if $self->debug;
my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
$_ = $';
@@ -969,87 +989,129 @@ sub decompose {
$struct->new_filter( $filter => $params, $negate );
}
- $last_type = '';
+ local $last_type = '';
} elsif ($self->modifier_count && /$modifier_re/) { # found a modifier
- warn "Encountered search modifier: $1\n" if $self->debug;
+ warn ' 'x$recursing."Encountered search modifier: $1\n" if $self->debug;
$_ = $';
if (!($struct->top_plan || $parser_config{$pkg}->{allow_nested_modifiers})) {
- warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
+ warn ' 'x$recursing." Search modifiers only allowed at the top level of the query\n" if $self->debug;
} else {
$struct->new_modifier($1);
}
- $last_type = '';
+ local $last_type = '';
} elsif ($self->modifier_count && /$modifier_as_class_re/) { # found a modifier
- warn "Encountered search modifier: $1\n" if $self->debug;
+ warn ' 'x$recursing."Encountered search modifier: $1\n" if $self->debug;
my $mod = $1;
$_ = $';
if (!($struct->top_plan || $parser_config{$pkg}->{allow_nested_modifiers})) {
- warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
+ warn ' 'x$recursing." Search modifiers only allowed at the top level of the query\n" if $self->debug;
} elsif ($2 =~ /^[ty1]/i) {
$struct->new_modifier($mod);
}
- $last_type = '';
+ local $last_type = '';
} elsif (/$float_start_re/) { # start of an explicit float
- warn "Encountered explicit float start\n" if $self->debug;
+ warn ' 'x$recursing."Encountered explicit float start\n" if $self->debug;
+ $floating = 1;
+ $fstart = $struct;
+
+ $last_class = $current_class;
+ $current_class = undef;
$self->floating_plan( $self->new_plan( floating => 1 ) ) if (!$self->floating_plan);
+
# pass the floating_plan struct to be modified by the float'ed chunk
- my ($floating_plan, $subremainder) = $self->new->decompose( $', undef, undef, undef, $self->floating_plan);
+ my ($floating_plan, $subremainder) = $self->new( debug => $self->debug )->decompose( $', undef, undef, undef, $self->floating_plan);
$_ = $subremainder;
+ warn ' 'x$recursing."Remainder after explicit float: $_\n" if $self->debug;
+
+ $current_class = $last_class;
$last_type = '';
} elsif (/$group_start_re/) { # start of an explicit group
- warn "Encountered explicit group start\n" if $self->debug;
+ warn ' 'x$recursing."Encountered explicit group start\n" if $self->debug;
my ($substruct, $subremainder) = $self->decompose( $', $current_class, $recursing + 1 );
$struct->add_node( $substruct ) if ($substruct);
$_ = $subremainder;
+ warn ' 'x$recursing."Query remainder after bool group: $_\n" if $self->debug;
+
+ local $last_type = '';
- $last_type = '';
} elsif (/$and_re/) { # ANDed expression
$_ = $';
- next if ($last_type eq 'AND');
- next if ($last_type eq 'OR');
- warn "Encountered AND\n" if $self->debug;
+ warn ' 'x$recursing."Encountered AND\n" if $self->debug;
+ do {warn ' 'x$recursing."!!! Already doing the bool dance for AND\n" if $self->debug; next} if ($last_type eq 'AND');
+ do {warn ' 'x$recursing."!!! Already doing the bool dance for OR\n" if $self->debug; next} if ($last_type eq 'OR');
+ local $last_type = 'AND';
+ warn ' 'x$recursing."Saving LHS, building RHS\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
+ #my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 1 );
$_ = $subremainder;
- $struct = $self->new_plan( level => $recursing, joiner => '&', floating => $LHS->floating );
+ warn ' 'x$recursing."RHS built\n" if $self->debug;
+ warn ' 'x$recursing."Post-AND remainder: $subremainder\n" if $self->debug;
+
+ my $wrapper = $self->new_plan( level => $recursing + 1 );
+
if ($LHS->floating) {
- $self->floating_plan($struct);
- $LHS->floating(0);
+ $wrapper->{query} = $LHS->{query};
+ my $outer_wrapper = $self->new_plan( level => $recursing + 1 );
+ $outer_wrapper->add_node($_) for ($wrapper,$RHS);
+ $LHS->{query} = [$outer_wrapper];
+ $struct = $LHS;
+ } else {
+ $wrapper->add_node($_) for ($LHS, $RHS);
+ $wrapper->plan_level($wrapper->plan_level); # reset levels all the way down
+ $struct = $self->new_plan( level => $recursing );
+ $struct->add_node($wrapper);
}
- $struct->add_node($_) for ($LHS, $RHS);
-
$self->parse_tree( $struct ) if ($self->parse_tree == $LHS);
- $last_type = 'AND';
+ local $last_type = '';
} elsif (/$or_re/) { # ORed expression
$_ = $';
- next if ($last_type eq 'AND');
- next if ($last_type eq 'OR');
- warn "Encountered OR\n" if $self->debug;
+ warn ' 'x$recursing."Encountered OR\n" if $self->debug;
+ do {warn ' 'x$recursing."!!! Already doing the bool dance for AND\n" if $self->debug; next} if ($last_type eq 'AND');
+ do {warn ' 'x$recursing."!!! Already doing the bool dance for OR\n" if $self->debug; next} if ($last_type eq 'OR');
+ local $last_type = 'OR';
+ warn ' 'x$recursing."Saving LHS, building RHS\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
+ #my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 2 );
$_ = $subremainder;
- $struct = $self->new_plan( level => $recursing, joiner => '|' );
- $struct->add_node($_) for ($LHS, $RHS);
+ warn ' 'x$recursing."RHS built\n" if $self->debug;
+ warn ' 'x$recursing."Post-OR remainder: $subremainder\n" if $self->debug;
+
+ my $wrapper = $self->new_plan( level => $recursing + 1, joiner => '|' );
+
+ if ($LHS->floating) {
+ $wrapper->{query} = $LHS->{query};
+ my $outer_wrapper = $self->new_plan( level => $recursing + 1, joiner => '|' );
+ $outer_wrapper->add_node($_) for ($wrapper,$RHS);
+ $LHS->{query} = [$outer_wrapper];
+ $struct = $LHS;
+ } else {
+ $wrapper->add_node($_) for ($LHS, $RHS);
+ $wrapper->plan_level($wrapper->plan_level); # reset levels all the way down
+ $struct = $self->new_plan( level => $recursing );
+ $struct->add_node($wrapper);
+ }
$self->parse_tree( $struct ) if ($self->parse_tree == $LHS);
- $last_type = 'OR';
+ local $last_type = '';
} elsif ($self->facet_class_count && /$facet_re/) { # changing current class
- warn "Encountered facet: $1$2 => $3\n" if $self->debug;
+ warn ' 'x$recursing."Encountered facet: $1$2 => $3\n" if $self->debug;
my $negate = ($1 eq $pkg->operator('disallowed')) ? 1 : 0;
my $facet = $2;
@@ -1057,34 +1119,34 @@ sub decompose {
$struct->new_facet( $facet => $facet_value, $negate );
$_ = $';
- $last_type = '';
+ local $last_type = '';
} elsif ($self->search_class_count && /$search_class_re/) { # changing current class
if ($last_type eq 'CLASS') {
$struct->remove_last_node( $current_class );
- warn "Encountered class change with no searches!\n" if $self->debug;
+ warn ' 'x$recursing."Encountered class change with no searches!\n" if $self->debug;
}
- warn "Encountered class change: $1\n" if $self->debug;
+ warn ' 'x$recursing."Encountered class change: $1\n" if $self->debug;
$current_class = $struct->classed_node( $1 )->requested_class();
$_ = $';
- $last_type = 'CLASS';
+ local $last_type = 'CLASS';
} elsif (/^\s*($required_re|$disallowed_re)?"([^"]+)"/) { # phrase, always anded
- warn 'Encountered' . ($1 ? " ['$1' modified]" : '') . " phrase: $2\n" if $self->debug;
+ warn ' 'x$recursing.'Encountered' . ($1 ? " ['$1' modified]" : '') . " phrase: $2\n" if $self->debug;
my $req_ness = $1 || '';
my $phrase = $2;
if (!$phrase_helper) {
- warn "Recursing into decompose with the phrase as a subquery\n" if $self->debug;
+ warn ' 'x$recursing."Recursing into decompose with the phrase as a subquery\n" if $self->debug;
my $after = $';
my ($substruct, $subremainder) = $self->decompose( qq/$req_ness"$phrase"/, $current_class, $recursing + 1, 1 );
$struct->add_node( $substruct ) if ($substruct);
$_ = $after;
} else {
- warn "Directly parsing the phrase subquery\n" if $self->debug;
+ warn ' 'x$recursing."Directly parsing the phrase subquery\n" if $self->debug;
$struct->joiner( '&' );
my $class_node = $struct->classed_node($current_class);
@@ -1101,41 +1163,38 @@ sub decompose {
}
- $last_type = '';
+ local $last_type = '';
+
+ } elsif (/^\s*$required_re([^${group_end}${float_end}\s"]+)/) { # phrase, always anded
+ warn ' 'x$recursing."Encountered required atom (mini phrase), transforming for phrase parse: $1\n" if $self->debug;
+
+ $_ = '"' . $1 . '"' . $';
-# } elsif (/^\s*$required_re([^\s"]+)/) { # phrase, always anded
-# warn "Encountered required atom (mini phrase): $1\n" if $self->debug;
-#
-# my $phrase = $1;
-#
-# my $class_node = $struct->classed_node($current_class);
-# $class_node->add_phrase( $phrase );
-# $_ = $phrase . $';
-# $struct->joiner( '&' );
-#
-# $last_type = '';
+ local $last_type = '';
} elsif (/^\s*([^${group_end}${float_end}\s]+)/o) { # atom
- warn "Encountered atom: $1\n" if $self->debug;
- warn "Remainder: $'\n" if $self->debug;
+ warn ' 'x$recursing."Encountered atom: $1\n" if $self->debug;
+ warn ' 'x$recursing."Remainder: $'\n" if $self->debug;
my $atom = $1;
my $after = $';
$_ = $after;
- $last_type = '';
+ local $last_type = '';
my $class_node = $struct->classed_node($current_class);
my $prefix = ($atom =~ s/^$disallowed_re//o) ? '!' : '';
my $truncate = ($atom =~ s/\*$//o) ? '*' : '';
- if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|','-','+')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
+ if ($atom ne '' and !grep { $atom =~ /^\Q$_\E+$/ } ('&','|')) { # throw away & and |, not allowed in tsquery, and not really useful anyway
# $class_node->add_phrase( $atom ) if ($atom =~ s/^$required_re//o);
# $class_node->add_unphrase( $atom ) if ($prefix eq '!');
$class_node->add_fts_atom( $atom, suffix => $truncate, prefix => $prefix, node => $class_node );
$struct->joiner( '&' );
}
+
+ local $last_type = '';
}
last unless ($_);
@@ -1277,6 +1336,7 @@ sub find_arrays_in_abstract {
#-------------------------------
package QueryParser::Canonicalize; # not OO
+use Data::Dumper;
sub _abstract_query2str_filter {
my $f = shift;
@@ -1305,6 +1365,7 @@ sub _kid_list {
return @{$$children{$op}};
}
+
# This should produce an equivalent query to the original, given an
# abstract_query.
sub abstract_query2str_impl {
@@ -1312,6 +1373,7 @@ sub abstract_query2str_impl {
my $depth = shift || 0;
my $qp_class ||= shift || 'QueryParser';
+ my $force_qp_node = shift || 0;
my $qpconfig = $QueryParser::parser_config{$qp_class};
my $fs = $qpconfig->{operators}{float_start};
@@ -1322,6 +1384,7 @@ sub abstract_query2str_impl {
my $or = $qpconfig->{operators}{or};
my $isnode = 0;
+ my $size = 0;
my $q = "";
if (exists $abstract_query->{type}) {
@@ -1331,8 +1394,10 @@ sub abstract_query2str_impl {
$q .= ($q ? ' ' : '') . join(" ", map { _abstract_query2str_modifier($_, $qp_class) } @{$abstract_query->{modifiers}}) if
exists $abstract_query->{modifiers};
- $isnode = 1
- if (!$abstract_query->{floating} && exists $abstract_query->{children} && _kid_list($abstract_query->{children}) > 1);
+
+ $size = _kid_list($abstract_query->{children});
+ $isnode = 1 if ($size > 1 and ($force_qp_node or $depth));
+ #warn "size: $size, depth: $depth, isnode: $isnode, AQ: ".Dumper($abstract_query);
} elsif ($abstract_query->{type} eq 'node') {
if ($abstract_query->{alias}) {
$q .= ($q ? ' ' : '') . $abstract_query->{alias};
@@ -1357,6 +1422,8 @@ sub abstract_query2str_impl {
}
}
+ my $next_depth = int($size > 1);
+
if (exists $abstract_query->{children}) {
my $op = (keys(%{$abstract_query->{children}}))[0];
@@ -1365,7 +1432,7 @@ sub abstract_query2str_impl {
my $sub_node = pop @{$abstract_query->{children}{$op}};
$abstract_query->{floating} = 0;
- $q = $fs . " " . abstract_query2str_impl($abstract_query,0,$qp_class) . $fe. " ";
+ $q = $fs . " " . abstract_query2str_impl($abstract_query,0,$qp_class, 1) . $fe. " ";
$abstract_query = $sub_node;
}
@@ -1375,7 +1442,7 @@ sub abstract_query2str_impl {
$q .= ($q ? ' ' : '') . join(
($op eq '&' ? ' ' : " $or "),
map {
- my $x = abstract_query2str_impl($_, $depth + 1, $qp_class); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
+ my $x = abstract_query2str_impl($_, $depth + $next_depth, $qp_class, $force_qp_node); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
} @{$abstract_query->{children}{$op}}
);
}
@@ -1384,7 +1451,7 @@ sub abstract_query2str_impl {
$q .= ($q ? ' ' : '') . join(
($op eq '&' ? ' ' : " $or "),
map {
- my $x = abstract_query2str_impl($_, $depth + 1, $qp_class); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
+ my $x = abstract_query2str_impl($_, $depth + $next_depth, $qp_class, $force_qp_node); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
} @{$abstract_query->{$op}}
);
}
@@ -1601,6 +1668,15 @@ sub top_plan {
sub plan_level {
my $self = shift;
+ my $level = shift;
+
+ if (defined $level) {
+ $self->{level} = $level;
+ for (@{$self->query_nodes}) {
+ $_->plan_level($level + 1) if (ref and $_->isa('QueryParser::query_plan'));
+ }
+ }
+
return $self->{level};
}
@@ -1681,6 +1757,7 @@ sub to_abstract_query {
my $abstract_query = {
type => "query_plan",
floating => $self->floating,
+ level => $self->plan_level,
filters => [map { $_->to_abstract_query } @{$self->filters}],
modifiers => [map { $_->to_abstract_query } @{$self->modifiers}]
};
@@ -1944,6 +2021,8 @@ sub to_abstract_query {
}
}
+ $abstract_query->{children} ||= { QueryParser::_util::default_joiner() => $kids };
+
if ($self->{phrases} and not $opts{no_phrases}) {
for my $phrase (@{$self->{phrases}}) {
# Phrases appear duplication in a real QP tree, and we don't want
commit 1cdbcb8eccbeec914aeeb05876cc44d164c2052c
Author: Jared Camins-Esakov <jcamins at cpbibliography.com>
Date: Fri Sep 7 22:44:50 2012 -0400
QueryParser unit test
Test that QueryParser can handle a variety of queries. This initial
unit test does the following:
1) Test the configuration of QueryParser.
2) Test that various queries have stable canonical representations.
3) Test that a number of equivalent queries are correctly parsed as
equivalent.
4) Test that a number of non-equivalent queries are correctly parsed as
NOT being equivalent.
5) Several other tests relating to query parsing.
This includes almost 100% subroutine coverage in the QueryParser class.
Other classes have somewhat lower test coverage.
As of 2012-09-09, several outstanding bugs affect these tests:
1) QueryParser->superpage cannot be unset.
2) Explicit groups are not handled correctly by the abstract query
to string converter.
3) There is no defined precedence between explicit boolean connectors
and implicit boolean connectors.
4) Modifiers are silently dropped when not at the top level of the query.
Signed-off-by: Jared Camins-Esakov <jcamins at cpbibliography.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/t/21-QueryParser.t b/Open-ILS/src/perlmods/t/21-QueryParser.t
new file mode 100644
index 0000000..2427999
--- /dev/null
+++ b/Open-ILS/src/perlmods/t/21-QueryParser.t
@@ -0,0 +1,297 @@
+#!perl
+
+use strict;
+use warnings; # FATAL => qw(all);
+use Test::More;
+
+BEGIN {
+ use_ok( 'OpenILS::Application::Storage::QueryParser' );
+# use_ok( 'OpenILS::Application::Storage::Driver::Pg::QueryParser' );
+}
+
+my %args = ( debug => 0 );
+my $QParser = QueryParser->new(%args);
+is(ref $QParser, 'QueryParser', 'Created QueryParser');
+is($QParser->operator('and'), '&&', 'Expected and operator');
+
+$Data::Dumper::Indent = 1;
+
+$QParser->add_search_class_alias( keyword => 'kw' );
+is ($QParser->search_class_count, 1, "Added one search class");
+init_qp();
+
+is ($QParser->search_class_count, 5, "Correct number of search classes");
+is (scalar(@{$QParser->search_fields()->{'author'}}), 3, "Correct number of search fields for 'author' class");
+$QParser->remove_search_field('author', 'personal');
+is (scalar(@{$QParser->search_fields()->{'author'}}), 2, "Removed search field");
+$QParser->remove_search_class('title');
+is ($QParser->search_class_count, 4, "Removed search class");
+is (scalar(@{$QParser->search_class_aliases->{'author'}}), 3, "Correct number of aliases for 'author' class");
+$QParser->remove_search_class_alias( author => 'au' );
+is (scalar(@{$QParser->search_class_aliases->{'author'}}), 2, "Removed alias for 'author' class");
+is (scalar(@{$QParser->search_field_aliases->{'subject'}->{'name'}}), 2, "Correct number of search field aliases for 'subject' class");
+$QParser->remove_search_field_alias( subject => name => 'nomen' );
+is (scalar(@{$QParser->search_field_aliases->{'subject'}->{'name'}}), 1, "Removed search field alias");
+
+is ($QParser->facet_class_count, 2, "Correct number of facet classes");
+is (scalar(@{$QParser->facet_fields()->{'author'}}), 2, "Correct number of facet fields for 'author' class");
+$QParser->remove_facet_field('author', 'personal');
+is (scalar(@{$QParser->facet_fields()->{'author'}}), 1, "Removed facet field");
+$QParser->remove_facet_class('author');
+is ($QParser->facet_class_count, 1, "Removed facet class");
+
+is ($QParser->filter_count, 28, "Correct number of filters");
+is (scalar(@{$QParser->filter_normalizers('skip_check')}), 0, 'No filter normalizers by default');
+$QParser->add_filter_normalizer('skip_check', \&test_filter_norm);
+is (scalar(@{$QParser->filter_normalizers('skip_check')}), 1, 'Added filter normalizer');
+is ($QParser->modifier_count, 8, "Correct number of modifiers");
+
+is_deeply ($QParser->custom_data('string'), { }, "No custom data set for 'string'");
+
+is($QParser->core_limit(25000), 25000, 'Core limit setting works');
+is($QParser->core_limit(), 25000, 'Core limit stays set');
+
+is($QParser->superpage(1), 1, 'Superpage setting works');
+is($QParser->superpage(), 1, 'Superpage stays set');
+is($QParser->superpage(0), 0, 'Superpage can be unset');
+
+is($QParser->superpage_size(1000), 1000, 'Superpage size setting works');
+is($QParser->superpage_size(), 1000, 'Superpage size stays set');
+
+init_qp();
+
+my %queries = (
+ '(keyword1 keyword2) || keyword3' => undef,
+ 'keyword1 || keyword2' => undef,
+ 'author:keyword1 keyword2' => undef,
+ '(keyword1) || (keyword2)' => undef,
+ 'keyword1 || keyword2 || keyword3' => undef,
+ '(keyword1 || keyword2) && keyword3' => undef,
+ 'keyword1 keyword2 || keyword3 keyword4' => sub {
+ my $query = shift;
+ # Unfortunately, the canonical representation of a query in master
+ # as of 2012/09/07 is not unambiguous
+ is($QParser->parse_tree()->to_abstract_query()->{children}->{'&'}, undef, "Outer-most operator in query {$query} is not AND");
+ is(ref $QParser->parse_tree()->to_abstract_query()->{children}->{'|'}, 'ARRAY', "Outer-most operator in query {$query} is OR");
+ },
+ 'keyword1 keyword2 && keyword3 keyword4' => undef,
+ 'keyword1 author:keyword2' => undef,
+ 'au:keyword1 kw:keyword2' => undef,
+ 'keyword1 pref_ou(lib)' => sub {
+ my $query = shift;
+ is($QParser->parse_tree->to_abstract_query()->{filters}->[0]->{name}, 'pref_ou', 'Generated filter for query');
+ },
+ 'keyword1 #available' => sub {
+ my $query = shift;
+ is($QParser->parse_tree->to_abstract_query()->{modifiers}->[0], 'available', 'Set modifier for query');
+ },
+ '(keyword1 keyword2) || keyword3 #available' => sub {
+ my $query = shift;
+ is($QParser->parse_tree->to_abstract_query()->{modifiers}->[0], 'available', 'Set modifier for query');
+ },
+ 'keyword1 testfilter(whatever)' => undef,
+ 'keyword1 sort:something' => undef,
+ '"phrase1 phrase2" keyword1' => undef, # NOTE: phrases do not have a stable canonical representation, 2012-09-09
+ 'keyword1 -keyword2' => undef,
+ 'keyword1 +keyword2' => undef,
+);
+
+my $query;
+my $testfunc;
+while (($query, $testfunc) = each (%queries)) {
+ init_qp();
+ $QParser->parse($query);
+ # TODO: Test initial parse
+ &$testfunc($query) if ($testfunc);
+ my $canonical = clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query()));
+ $canonical = reparse($canonical);
+ init_qp();
+ $QParser->parse($canonical);
+ is(clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query())), $canonical, "Building query from canonical query is idempotent for query {$query}");
+}
+
+my %equivalences = (
+ 'keyword1 keyword2' => 'keyword1 && keyword2',
+ 'keyword1 keyword2 || keyword3 keyword4' => 'keyword1 && keyword2 || keyword3 && keyword4',
+ 'keyword1 keyword2 || keyword3 keyword4' => '(keyword1 keyword2) || (keyword3 keyword4)',
+ 'keyword1 keyword2 && keyword3 keyword4' => '(keyword1 && keyword2) && (keyword3 && keyword4)',
+ 'keyword1 || && keyword2' => 'keyword1 || keyword2',
+ 'keyword1' => 'keyword:keyword1',
+);
+
+my $equivalent;
+while (($query, $equivalent) = each (%equivalences)) {
+ init_qp();
+ $QParser->parse($query);
+ my $canonical1 = reparse(clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query())));
+ init_qp();
+ $QParser->parse($equivalent);
+ my $canonical2 = reparse(clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query())));
+ is($canonical1, $canonical2, "Queries {$query} and {$equivalent} are equivalent");
+}
+
+my %differences = (
+ '(keyword1 keyword2) || keyword3' => 'keyword1 && (keyword2 || keyword3)',
+ 'keyword1 || (keyword2 && keyword3)' => '(keyword1 || keyword2) && keyword3',
+ '(keyword1 || keyword2) && keyword3' => 'keyword1 || (keyword2 && keyword3)',
+ 'keyword1 keyword2 || keyword3 keyword4' => '(keyword1 keyword2 || keyword3) keyword4', # this should fail on master, 2012-09-07
+);
+
+
+my $different;
+while (($query, $different) = each (%differences)) {
+ init_qp();
+ $QParser->parse($query);
+ my $canonical1 = reparse(clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query())));
+ init_qp();
+ $QParser->parse($different);
+ my $canonical2 = reparse(clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query())));
+ isnt($canonical1, $canonical2, "Queries {$query} and {$different} are not equivalent");
+}
+
+
+done_testing;
+
+sub test_filter_norm {
+ return;
+}
+
+sub test_filter_callback {
+ my ($QParser, $struct, $filter, $params, $negate) = @_;
+ is($filter, 'testfilter', 'Filter callback on correct filter');
+ return;
+}
+
+sub clean {
+ my $string = shift;
+ $string =~ s/\s+/ /g;
+ $string =~ s/ \)/\)/g;
+ $string =~ s/\( /\(/g;
+ $string =~ s/ $//g;
+ $string =~ s/^ //g;
+
+ ($string, undef) = parse_parens($string);
+
+ $string =~ s/(^| )\(([^) ]+)\)/$2/g;
+ $string =~ s/^\(([^)]*)\)$/$1/g;
+
+ return $string;
+}
+
+sub parse_parens {
+ my $string = shift;
+ my $subres;
+ my $result = '';
+ while (my $nextchar = substr($string, 0, 1)) {
+ $string = substr($string, 1);
+ if ($nextchar eq '(') {
+ ($subres, $string) = parse_parens($string);
+ if ($result || ! (substr($string, 0, 1) eq ')')) {
+ $result .= "($subres)";
+ } else {
+ $result = $subres;
+ }
+ } elsif ($nextchar eq ')') {
+ return ($result, $string);
+ } else {
+ $result .= $nextchar;
+ }
+ }
+ return $result;
+}
+
+sub reparse {
+ my $canonical = shift;
+ my $repeats = $canonical =~ tr/&/&/;
+ $repeats = ($repeats / 2) + 1;
+ my $result;
+ while (--$repeats) {
+ init_qp();
+ $QParser->parse($canonical);
+ $canonical = clean(QueryParser::Canonicalize::abstract_query2str_impl($QParser->parse_tree()->to_abstract_query()));
+ }
+ return $canonical;
+}
+
+sub init_qp {
+ $QueryParser::parser_config{QueryParser}->{allow_nested_modifiers} = 1;
+ $QParser = QueryParser->new(%args);
+ $QParser->add_search_class_alias( title => 'ti' );
+ $QParser->add_search_class_alias( author => 'au' );
+ $QParser->add_search_class_alias( author => 'name' );
+ $QParser->add_search_class_alias( author => 'dc.contributor' );
+ $QParser->add_search_class_alias( subject => 'su' );
+ $QParser->add_search_class_alias( subject => 'bib.subject(?:Title|Place|Occupation)' );
+ $QParser->add_search_class_alias( series => 'se' );
+ $QParser->add_search_class_alias( keyword => 'dc.identifier' );
+
+ $QParser->add_query_normalizer( author => corporate => 'search_normalize' );
+ $QParser->add_query_normalizer( keyword => keyword => 'search_normalize' );
+
+ $QParser->add_search_field_alias( subject => name => 'bib.subjectName' );
+ $QParser->add_search_field_alias( subject => name => 'nomen' );
+
+ $QParser->add_search_field( 'author' => 'personal' );
+ $QParser->add_search_field( 'author' => 'corporate' );
+ $QParser->add_search_field( 'author' => 'meeting' );
+
+ $QParser->default_search_class( 'keyword' );
+
+ # will be retained simply for back-compat
+ $QParser->add_search_filter( 'format' );
+
+ # grumble grumble, special cases against date1 and date2
+ $QParser->add_search_filter( 'before' );
+ $QParser->add_search_filter( 'after' );
+ $QParser->add_search_filter( 'between' );
+ $QParser->add_search_filter( 'during' );
+
+ # used by layers above this
+ $QParser->add_search_filter( 'statuses' );
+ $QParser->add_search_filter( 'locations' );
+ $QParser->add_search_filter( 'location_groups' );
+ $QParser->add_search_filter( 'site' );
+ $QParser->add_search_filter( 'pref_ou' );
+ $QParser->add_search_filter( 'lasso' );
+ $QParser->add_search_filter( 'my_lasso' );
+ $QParser->add_search_filter( 'depth' );
+ $QParser->add_search_filter( 'language' );
+ $QParser->add_search_filter( 'offset' );
+ $QParser->add_search_filter( 'limit' );
+ $QParser->add_search_filter( 'check_limit' );
+ $QParser->add_search_filter( 'skip_check' );
+ $QParser->add_search_filter( 'superpage' );
+ $QParser->add_search_filter( 'estimation_strategy' );
+ $QParser->add_search_modifier( 'available' );
+ $QParser->add_search_modifier( 'staff' );
+
+ # Start from container data (bre, acn, acp): container(bre,bookbag,123,deadb33fdeadb33fdeadb33fdeadb33f)
+ $QParser->add_search_filter( 'container' );
+
+ # Start from a list of record ids, either bre or metarecords, depending on the #metabib modifier
+ $QParser->add_search_filter( 'record_list' );
+
+ # used internally, but generally not user-settable
+ $QParser->add_search_filter( 'preferred_language' );
+ $QParser->add_search_filter( 'preferred_language_weight' );
+ $QParser->add_search_filter( 'preferred_language_multiplier' );
+ $QParser->add_search_filter( 'core_limit' );
+
+ # XXX Valid values to be supplied by SVF
+ $QParser->add_search_filter( 'sort' );
+
+ # modifies core query, not configurable
+ $QParser->add_search_modifier( 'descending' );
+ $QParser->add_search_modifier( 'ascending' );
+ $QParser->add_search_modifier( 'nullsfirst' );
+ $QParser->add_search_modifier( 'nullslast' );
+ $QParser->add_search_modifier( 'metarecord' );
+ $QParser->add_search_modifier( 'metabib' );
+
+ $QParser->add_facet_field( 'author' => 'personal' );
+ $QParser->add_facet_field( 'author' => 'corporate' );
+ $QParser->add_facet_field( 'subject' => 'topic' );
+ $QParser->add_facet_field( 'subject' => 'geographic' );
+
+ $QParser->add_search_filter( 'testfilter', \&test_filter_callback );
+}
commit ba2ad7bda934b2184eba42dcd1eb1860bbcd6599
Author: Jared Camins-Esakov <jcamins at cpbibliography.com>
Date: Sun Sep 9 10:52:06 2012 -0400
Start adding skeletal POD for subroutines
Signed-off-by: Jared Camins-Esakov <jcamins at cpbibliography.com>
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 0be69cb..4746d78 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -3,6 +3,25 @@ use warnings;
package QueryParser;
use OpenSRF::Utils::JSON;
+
+=head1 NAME
+
+QueryParser - basic QueryParser class
+
+=head1 SYNOPSIS
+
+use QueryParser;
+my $QParser = QueryParser->new(%args);
+
+=head1 DESCRIPTION
+
+Main entrypoint into the QueryParser functionality.
+
+=head1 FUNCTIONS
+
+=cut
+
+# Note that the first key must match the name of the package.
our %parser_config = (
QueryParser => {
filters => [],
@@ -29,26 +48,51 @@ sub canonicalize {
}
+=head2 facet_class_count
+
+ $count = $QParser->facet_class_count();
+=cut
+
sub facet_class_count {
my $self = shift;
return @{$self->facet_classes};
}
+=head2 search_class_count
+
+ $count = $QParser->search_class_count();
+=cut
+
sub search_class_count {
my $self = shift;
return @{$self->search_classes};
}
+=head2 filter_count
+
+ $count = $QParser->filter_count();
+=cut
+
sub filter_count {
my $self = shift;
return @{$self->filters};
}
+=head2 modifier_count
+
+ $count = $QParser->modifier_count();
+=cut
+
sub modifier_count {
my $self = shift;
return @{$self->modifiers};
}
+=head2 custom_data
+
+ $data = $QParser->custom_data($class);
+=cut
+
sub custom_data {
my $class = shift;
$class = ref($class) || $class;
@@ -57,6 +101,13 @@ sub custom_data {
return $parser_config{$class}{custom_data};
}
+=head2 operators
+
+ $operators = $QParser->operators();
+
+Returns hashref of the configured operators.
+=cut
+
sub operators {
my $class = shift;
$class = ref($class) || $class;
@@ -74,6 +125,13 @@ sub allow_nested_modifiers {
return $parser_config{$class}{allow_nested_modifiers};
}
+=head2 filters
+
+ $filters = $QParser->filters();
+
+Returns arrayref of the configured filters.
+=cut
+
sub filters {
my $class = shift;
$class = ref($class) || $class;
@@ -82,6 +140,13 @@ sub filters {
return $parser_config{$class}{filters};
}
+=head2 filter_callbacks
+
+ $filter_callbacks = $QParser->filter_callbacks();
+
+Returns hashref of the configured filter callbacks.
+=cut
+
sub filter_callbacks {
my $class = shift;
$class = ref($class) || $class;
@@ -90,6 +155,13 @@ sub filter_callbacks {
return $parser_config{$class}{filter_callbacks};
}
+=head2 modifiers
+
+ $modifiers = $QParser->modifiers();
+
+Returns arrayref of the configured modifiers.
+=cut
+
sub modifiers {
my $class = shift;
$class = ref($class) || $class;
@@ -98,6 +170,13 @@ sub modifiers {
return $parser_config{$class}{modifiers};
}
+=head2 new
+
+ $QParser = QueryParser->new(%args);
+
+Creates a new QueryParser object.
+=cut
+
sub new {
my $class = shift;
$class = ref($class) || $class;
@@ -117,12 +196,27 @@ sub new {
return $self;
}
+=head2 new_plan
+
+ $query_plan = $QParser->new_plan();
+
+Create a new query plan.
+=cut
+
sub new_plan {
my $self = shift;
my $pkg = ref($self) || $self;
return do{$pkg.'::query_plan'}->new( QueryParser => $self, @_ );
}
+=head2 add_search_filter
+
+ $QParser->add_search_filter($filter, [$callback]);
+
+Adds a filter with the specified name and an optional callback to the
+QueryParser configuration.
+=cut
+
sub add_search_filter {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -135,6 +229,13 @@ sub add_search_filter {
return $filter;
}
+=head2 add_search_modifier
+
+ $QParser->add_search_modifier($modifier);
+
+Adds a modifier with the specified name to the QueryParser configuration.
+=cut
+
sub add_search_modifier {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -145,6 +246,13 @@ sub add_search_modifier {
return $modifier;
}
+=head2 add_facet_class
+
+ $QParser->add_facet_class($facet_class);
+
+Adds a facet class with the specified name to the QueryParser configuration.
+=cut
+
sub add_facet_class {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -158,6 +266,13 @@ sub add_facet_class {
return $class;
}
+=head2 add_search_class
+
+ $QParser->add_search_class($class);
+
+Adds a search class with the specified name to the QueryParser configuration.
+=cut
+
sub add_search_class {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -172,6 +287,33 @@ sub add_search_class {
return $class;
}
+=head2 add_search_modifier
+
+ $op = $QParser->operator($operator, [$newvalue]);
+
+Retrieves or sets value for the specified operator. Valid operators and
+their defaults are as follows:
+
+=over 4
+
+=item * and => &&
+
+=item * or => ||
+
+=item * group_start => (
+
+=item * group_end => )
+
+=item * required => +
+
+=item * disallowed => -
+
+=item * modifier => #
+
+=back
+
+=cut
+
sub operator {
my $class = shift;
$class = ref($class) || $class;
@@ -186,6 +328,14 @@ sub operator {
return $parser_config{$class}{operators}{$opname};
}
+=head2 facet_classes
+
+ $classes = $QParser->facet_classes([\@newclasses]);
+
+Returns arrayref of all configured facet classes after optionally
+replacing configuration.
+=cut
+
sub facet_classes {
my $class = shift;
$class = ref($class) || $class;
@@ -196,6 +346,14 @@ sub facet_classes {
return $parser_config{$class}{facet_classes};
}
+=head2 search_classes
+
+ $classes = $QParser->search_classes([\@newclasses]);
+
+Returns arrayref of all configured search classes after optionally
+replacing the previous configuration.
+=cut
+
sub search_classes {
my $class = shift;
$class = ref($class) || $class;
@@ -206,6 +364,12 @@ sub search_classes {
return $parser_config{$class}{classes};
}
+=head2 add_query_normalizer
+
+ $function = $QParser->add_query_normalizer($class, $field, $func, [\@params]);
+
+=cut
+
sub add_query_normalizer {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -225,6 +389,14 @@ sub add_query_normalizer {
return $func;
}
+=head2 query_normalizers
+
+ $normalizers = $QParser->query_normalizers($class, $field);
+
+Returns a list of normalizers associated with the specified search class
+and field
+=cut
+
sub query_normalizers {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -245,6 +417,13 @@ sub query_normalizers {
return $parser_config{$pkg}{normalizers};
}
+=head2 add_filter_normalizer
+
+ $normalizer = $QParser->add_filter_normalizer($filter, $func, [\@params]);
+
+Adds a normalizer function to the specified filter.
+=cut
+
sub add_filter_normalizer {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -259,6 +438,13 @@ sub add_filter_normalizer {
return $func;
}
+=head2 filter_normalizers
+
+ $normalizers = $QParser->filter_normalizers($filter);
+
+Return arrayref of normalizer functions associated with the specified filter.
+=cut
+
sub filter_normalizers {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -274,6 +460,13 @@ sub filter_normalizers {
return $parser_config{$pkg}{filter_normalizers};
}
+=head2 default_search_class
+
+ $default_class = $QParser->default_search_class([$class]);
+
+Set or return the default search class.
+=cut
+
sub default_search_class {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -283,6 +476,13 @@ sub default_search_class {
return $QueryParser::parser_config{$pkg}{default_class};
}
+=head2 remove_facet_class
+
+ $QParser->remove_facet_class($class);
+
+Remove the specified facet class from the configuration.
+=cut
+
sub remove_facet_class {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -296,6 +496,13 @@ sub remove_facet_class {
return $class;
}
+=head2 remove_search_class
+
+ $QParser->remove_search_class($class);
+
+Remove the specified search class from the configuration.
+=cut
+
sub remove_search_class {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -309,6 +516,14 @@ sub remove_search_class {
return $class;
}
+=head2 add_facet_field
+
+ $QParser->add_facet_field($class, $field);
+
+Adds the specified field (and facet class if it doesn't already exist)
+to the configuration.
+=cut
+
sub add_facet_field {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -324,6 +539,13 @@ sub add_facet_field {
return { $class => $field };
}
+=head2 facet_fields
+
+ $fields = $QParser->facet_fields($class);
+
+Returns arrayref with list of fields for specified facet class.
+=cut
+
sub facet_fields {
my $class = shift;
$class = ref($class) || $class;
@@ -332,6 +554,14 @@ sub facet_fields {
return $parser_config{$class}{facet_fields};
}
+=head2 add_search_field
+
+ $QParser->add_search_field($class, $field);
+
+Adds the specified field (and facet class if it doesn't already exist)
+to the configuration.
+=cut
+
sub add_search_field {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -347,6 +577,13 @@ sub add_search_field {
return { $class => $field };
}
+=head2 search_fields
+
+ $fields = $QParser->search_fields();
+
+Returns arrayref with list of configured search fields.
+=cut
+
sub search_fields {
my $class = shift;
$class = ref($class) || $class;
@@ -355,6 +592,11 @@ sub search_fields {
return $parser_config{$class}{fields};
}
+=head2 add_search_class_alias
+
+ $QParser->add_search_class_alias($class, $alias);
+=cut
+
sub add_search_class_alias {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -370,6 +612,11 @@ sub add_search_class_alias {
return { $class => $alias };
}
+=head2 search_class_aliases
+
+ $aliases = $QParser->search_class_aliases($class);
+=cut
+
sub search_class_aliases {
my $class = shift;
$class = ref($class) || $class;
@@ -378,6 +625,11 @@ sub search_class_aliases {
return $parser_config{$class}{class_map};
}
+=head2 add_search_field_alias
+
+ $QParser->add_search_field_alias($class, $field, $alias);
+=cut
+
sub add_search_field_alias {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -392,6 +644,11 @@ sub add_search_field_alias {
return { $class => { $field => $alias } };
}
+=head2 search_field_aliases
+
+ $aliases = $QParser->search_field_aliases();
+=cut
+
sub search_field_aliases {
my $class = shift;
$class = ref($class) || $class;
@@ -400,6 +657,11 @@ sub search_field_aliases {
return $parser_config{$class}{field_alias_map};
}
+=head2 remove_facet_field
+
+ $QParser->remove_facet_field($class, $field);
+=cut
+
sub remove_facet_field {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -413,6 +675,11 @@ sub remove_facet_field {
return { $class => $field };
}
+=head2 remove_search_field
+
+ $QParser->remove_search_field($class, $field);
+=cut
+
sub remove_search_field {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -426,6 +693,11 @@ sub remove_search_field {
return { $class => $field };
}
+=head2 remove_search_field_alias
+
+ $QParser->remove_search_field_alias($class, $field, $alias);
+=cut
+
sub remove_search_field_alias {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -440,6 +712,11 @@ sub remove_search_field_alias {
return { $class => { $field => $alias } };
}
+=head2 remove_search_class_alias
+
+ $QParser->remove_search_class_alias($class, $alias);
+=cut
+
sub remove_search_class_alias {
my $pkg = shift;
$pkg = ref($pkg) || $pkg;
@@ -453,6 +730,13 @@ sub remove_search_class_alias {
return { $class => $alias };
}
+=head2 debug
+
+ $debug = $QParser->debug([$debug]);
+
+Return or set whether debugging output is enabled.
+=cut
+
sub debug {
my $self = shift;
my $q = shift;
@@ -460,6 +744,13 @@ sub debug {
return $self->{_debug};
}
+=head2 query
+
+ $query = $QParser->query([$query]);
+
+Return or set the query.
+=cut
+
sub query {
my $self = shift;
my $q = shift;
@@ -467,6 +758,13 @@ sub query {
return $self->{_query};
}
+=head2 parse_tree
+
+ $parse_tree = $QParser->parse_tree([$parse_tree]);
+
+Return or set the parse tree associated with the QueryParser.
+=cut
+
sub parse_tree {
my $self = shift;
my $q = shift;
@@ -481,6 +779,14 @@ sub floating_plan {
return $self->{_top};
}
+=head2 parse
+
+ $QParser->parse([$query]);
+
+Parse the specified query, or the query already associated with the QueryParser
+object.
+=cut
+
sub parse {
my $self = shift;
my $pkg = ref($self) || $self;
@@ -500,6 +806,15 @@ sub parse {
return $self;
}
+=head2 decompose
+
+ ($struct, $remainder) = $QParser->decompose($querystring, [$current_class], [$recursing], [$phrase_helper]);
+
+This routine does the heavy work of parsing the query string recursively.
+Returns the top level query plan, or the query plan from a lower level plus
+the portion of the query string that needs to be processed at a higher level.
+=cut
+
sub decompose {
my $self = shift;
my $pkg = ref($self) || $self;
@@ -836,6 +1151,11 @@ sub decompose {
return ($struct, $remainder);
}
+=head2 find_class_index
+
+ $index = $QParser->find_class_index($class, $query);
+=cut
+
sub find_class_index {
my $class = shift;
my $query = shift;
@@ -852,6 +1172,13 @@ sub find_class_index {
return -1;
}
+=head2 core_limit
+
+ $limit = $QParser->core_limit([$limit]);
+
+Return and/or set the core_limit.
+=cut
+
sub core_limit {
my $self = shift;
my $l = shift;
@@ -859,6 +1186,13 @@ sub core_limit {
return $self->{core_limit};
}
+=head2 superpage
+
+ $superpage = $QParser->superpage([$superpage]);
+
+Return and/or set the superpage.
+=cut
+
sub superpage {
my $self = shift;
my $l = shift;
@@ -866,6 +1200,13 @@ sub superpage {
return $self->{superpage};
}
+=head2 superpage_size
+
+ $size = $QParser->superpage_size([$size]);
+
+Return and/or set the superpage size.
+=cut
+
sub superpage_size {
my $self = shift;
my $l = shift;
commit be608c694172d6536b8d48efea5bae4c338fdca6
Author: Mike Rylander <mrylander at gmail.com>
Date: Mon Sep 10 15:31:05 2012 -0400
Move allow_nested_modifiers to the driver level, provide a wrapper for it, and add that to the Pg test setup as an example
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
index cbfd99c..d566a37 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm
@@ -380,6 +380,8 @@ sub initialize {
sub TEST_SETUP {
+ __PACKAGE__->allow_nested_modifiers(1);
+
__PACKAGE__->add_search_field_id_map( series => seriestitle => 1 => 1 );
__PACKAGE__->add_search_field_id_map( series => seriestitle => 1 => 1 );
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 6f948ef..0be69cb 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -65,6 +65,15 @@ sub operators {
return $parser_config{$class}{operators};
}
+sub allow_nested_modifiers {
+ my $class = shift;
+ my $v = shift;
+ $class = ref($class) || $class;
+
+ $parser_config{$class}{allow_nested_modifiers} = $v if (defined $v);
+ return $parser_config{$class}{allow_nested_modifiers};
+}
+
sub filters {
my $class = shift;
$class = ref($class) || $class;
@@ -650,7 +659,7 @@ sub decompose {
warn "Encountered search modifier: $1\n" if $self->debug;
$_ = $';
- if (!($struct->top_plan || $parser_config{QueryParser}->{allow_nested_modifiers})) {
+ if (!($struct->top_plan || $parser_config{$pkg}->{allow_nested_modifiers})) {
warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
} else {
$struct->new_modifier($1);
@@ -663,7 +672,7 @@ sub decompose {
my $mod = $1;
$_ = $';
- if (!($struct->top_plan || $parser_config{QueryParser}->{allow_nested_modifiers})) {
+ if (!($struct->top_plan || $parser_config{$pkg}->{allow_nested_modifiers})) {
warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
} elsif ($2 =~ /^[ty1]/i) {
$struct->new_modifier($mod);
commit 264a90828359118a3b736e8de4a14a450997b4eb
Author: Jared Camins-Esakov <jcamins at cpbibliography.com>
Date: Fri Sep 7 23:36:36 2012 -0400
Allow nested modifiers
Signed-off-by: Jared Camins-Esakov <jcamins at cpbibliography.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 0b756de..6f948ef 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -650,7 +650,7 @@ sub decompose {
warn "Encountered search modifier: $1\n" if $self->debug;
$_ = $';
- if (!$struct->top_plan) {
+ if (!($struct->top_plan || $parser_config{QueryParser}->{allow_nested_modifiers})) {
warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
} else {
$struct->new_modifier($1);
@@ -663,7 +663,7 @@ sub decompose {
my $mod = $1;
$_ = $';
- if (!$struct->top_plan) {
+ if (!($struct->top_plan || $parser_config{QueryParser}->{allow_nested_modifiers})) {
warn " Search modifiers only allowed at the top level of the query\n" if $self->debug;
} elsif ($2 =~ /^[ty1]/i) {
$struct->new_modifier($mod);
commit 1d5ed2a3a1d6eba163d6a92866b2cdeef8ad5165
Author: Mike Rylander <mrylander at gmail.com>
Date: Mon Sep 10 14:58:01 2012 -0400
Pretty-fy canonicalization
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 27cff0d..0b756de 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -454,7 +454,7 @@ sub debug {
sub query {
my $self = shift;
my $q = shift;
- $self->{_query} = $q if (defined $q);
+ $self->{_query} = " $q " if (defined $q);
return $self->{_query};
}
@@ -694,10 +694,15 @@ sub decompose {
warn "Encountered AND\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( $group_start.$_.$group_end, $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
$_ = $subremainder;
- $struct = $self->new_plan( level => $recursing, joiner => '&' );
+ $struct = $self->new_plan( level => $recursing, joiner => '&', floating => $LHS->floating );
+ if ($LHS->floating) {
+ $self->floating_plan($struct);
+ $LHS->floating(0);
+ }
+
$struct->add_node($_) for ($LHS, $RHS);
$self->parse_tree( $struct ) if ($self->parse_tree == $LHS);
@@ -710,7 +715,7 @@ sub decompose {
warn "Encountered OR\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( $group_start.$_.$group_end, $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( "$group_start $_ $group_end", $current_class, $recursing + 1 );
$_ = $subremainder;
$struct = $self->new_plan( level => $recursing, joiner => '|' );
@@ -785,7 +790,7 @@ sub decompose {
# $struct->joiner( '&' );
#
# $last_type = '';
- } elsif (/^\s*([^$group_end\s]+)/o && /^\s*([^$float_end\s]+)/o) { # atom
+ } elsif (/^\s*([^${group_end}${float_end}\s]+)/o) { # atom
warn "Encountered atom: $1\n" if $self->debug;
warn "Remainder: $'\n" if $self->debug;
@@ -944,6 +949,12 @@ sub _abstract_query2str_modifier {
return $qpconfig->{operators}{modifier} . $f;
}
+sub _kid_list {
+ my $children = shift;
+ my $op = (keys %$children)[0];
+ return @{$$children{$op}};
+}
+
# This should produce an equivalent query to the original, given an
# abstract_query.
sub abstract_query2str_impl {
@@ -960,42 +971,39 @@ sub abstract_query2str_impl {
my $and = $qpconfig->{operators}{and};
my $or = $qpconfig->{operators}{or};
- my $needs_group = 0;
+ my $isnode = 0;
my $q = "";
if (exists $abstract_query->{type}) {
if ($abstract_query->{type} eq 'query_plan') {
$q .= join(" ", map { _abstract_query2str_filter($_, $qp_class) } @{$abstract_query->{filters}}) if
exists $abstract_query->{filters};
- $needs_group += scalar(@{$abstract_query->{filters}}) if exists $abstract_query->{filters};
-
- $q .= " ";
- $q .= join(" ", map { _abstract_query2str_modifier($_, $qp_class) } @{$abstract_query->{modifiers}}) if
+ $q .= ($q ? ' ' : '') . join(" ", map { _abstract_query2str_modifier($_, $qp_class) } @{$abstract_query->{modifiers}}) if
exists $abstract_query->{modifiers};
- $needs_group += scalar(@{$abstract_query->{modifiers}}) if exists $abstract_query->{modifiers};
+ $isnode = 1
+ if (!$abstract_query->{floating} && exists $abstract_query->{children} && _kid_list($abstract_query->{children}) > 1);
} elsif ($abstract_query->{type} eq 'node') {
if ($abstract_query->{alias}) {
- $q .= " " . $abstract_query->{alias};
+ $q .= ($q ? ' ' : '') . $abstract_query->{alias};
$q .= "|$_" foreach @{$abstract_query->{alias_fields}};
} else {
- $q .= " " . $abstract_query->{class};
+ $q .= ($q ? ' ' : '') . $abstract_query->{class};
$q .= "|$_" foreach @{$abstract_query->{fields}};
}
$q .= ":";
+ $isnode = 1;
} elsif ($abstract_query->{type} eq 'atom') {
my $prefix = $abstract_query->{prefix} || '';
$prefix = $qpconfig->{operators}{disallowed} if $prefix eq '!';
- $q .= $prefix .
+ $q .= ($q ? ' ' : '') . $prefix .
($abstract_query->{content} || '') .
($abstract_query->{suffix} || '');
- $needs_group += 1;
} elsif ($abstract_query->{type} eq 'facet') {
# facet syntax [ # ] is hardcoded I guess?
my $prefix = $abstract_query->{negate} ? $qpconfig->{operators}{disallowed} : '';
- $q .= $prefix . $abstract_query->{name} . "[" .
+ $q .= ($q ? ' ' : '') . $prefix . $abstract_query->{name} . "[" .
join(" # ", @{$abstract_query->{values}}) . "]";
- $needs_group += 1;
}
}
@@ -1007,34 +1015,31 @@ sub abstract_query2str_impl {
my $sub_node = pop @{$abstract_query->{children}{$op}};
$abstract_query->{floating} = 0;
- $q = $fs.abstract_query2str_impl($abstract_query,0,$qp_class).$fe;
+ $q = $fs . " " . abstract_query2str_impl($abstract_query,0,$qp_class) . $fe. " ";
$abstract_query = $sub_node;
}
if ($abstract_query && exists $abstract_query->{children}) {
$op = (keys(%{$abstract_query->{children}}))[0];
- $q .= join(
- " " . ($op eq '&' ? '' : $or) . " ",
+ $q .= ($q ? ' ' : '') . join(
+ ($op eq '&' ? ' ' : " $or "),
map {
- abstract_query2str_impl($_, $depth + 1, $qp_class)
+ my $x = abstract_query2str_impl($_, $depth + 1, $qp_class); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
} @{$abstract_query->{children}{$op}}
);
- $needs_group += scalar(@{$abstract_query->{children}{$op}});
}
} elsif ($abstract_query->{'&'} or $abstract_query->{'|'}) {
my $op = (keys(%{$abstract_query}))[0];
- $q .= join(
- " " . ($op eq '&' ? '' : $or) . " ",
+ $q .= ($q ? ' ' : '') . join(
+ ($op eq '&' ? ' ' : " $or "),
map {
- abstract_query2str_impl($_, $depth + 1, $qp_class)
+ my $x = abstract_query2str_impl($_, $depth + 1, $qp_class); $x =~ s/^\s+//; $x =~ s/\s+$//; $x;
} @{$abstract_query->{$op}}
);
- $needs_group += scalar(@{$abstract_query->{$op}});
}
- $q .= " ";
- $q = $gs . $q . $ge if ($needs_group > 1 and $depth);
+ $q = "$gs$q$ge" if ($isnode);
return $q;
}
@@ -1220,6 +1225,13 @@ sub query_nodes {
return $self->{query};
}
+sub floating {
+ my $self = shift;
+ my $f = shift;
+ $self->{floating} = $f if (defined $f);
+ return $self->{floating};
+}
+
sub add_node {
my $self = shift;
my $node = shift;
@@ -1318,7 +1330,7 @@ sub to_abstract_query {
my $abstract_query = {
type => "query_plan",
- floating => $self->{floating},
+ floating => $self->floating,
filters => [map { $_->to_abstract_query } @{$self->filters}],
modifiers => [map { $_->to_abstract_query } @{$self->modifiers}]
};
commit cdb64b8159ec7edf920bf86dfef2fad96fe12fdf
Author: Mike Rylander <mrylander at gmail.com>
Date: Mon Sep 10 13:21:30 2012 -0400
Teach QP about floating (force-to-top) subplans indicated by {{...}}
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 3838dd3..27cff0d 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -10,6 +10,8 @@ our %parser_config = (
operators => {
'and' => '&&',
'or' => '||',
+ float_start => '{{',
+ float_end => '}}',
group_start => '(',
group_end => ')',
required => '+',
@@ -463,16 +465,29 @@ sub parse_tree {
return $self->{_parse_tree};
}
+sub floating_plan {
+ my $self = shift;
+ my $q = shift;
+ $self->{_top} = $q if (defined $q);
+ return $self->{_top};
+}
+
sub parse {
my $self = shift;
my $pkg = ref($self) || $self;
warn " ** parse package is $pkg\n" if $self->debug;
- $self->parse_tree(
- $self->decompose(
- $self->query( shift() )
- )
- );
+# $self->parse_tree(
+# $self->decompose(
+# $self->query( shift() )
+# )
+# );
+
+ $self->decompose( $self->query( shift() ) );
+ if ($self->floating_plan) {
+ $self->floating_plan->add_node( $self->parse_tree );
+ $self->parse_tree( $self->floating_plan );
+ }
return $self;
}
@@ -547,12 +562,18 @@ sub decompose {
my $or_re = $pkg->operator('or');
$or_re = qr/^\s*\Q$or_re\E/;
- my $group_start_re = $pkg->operator('group_start');
- $group_start_re = qr/^\s*\Q$group_start_re\E/;
+ my $group_start = $pkg->operator('group_start');
+ my $group_start_re = qr/^\s*\Q$group_start\E/;
my $group_end = $pkg->operator('group_end');
my $group_end_re = qr/^\s*\Q$group_end\E/;
+ my $float_start = $pkg->operator('float_start');
+ my $float_start_re = qr/^\s*\Q$float_start\E/;
+
+ my $float_end = $pkg->operator('float_end');
+ my $float_end_re = qr/^\s*\Q$float_end\E/;
+
my $modifier_tag_re = $pkg->operator('modifier');
$modifier_tag_re = qr/^\s*\Q$modifier_tag_re\E/;
@@ -567,13 +588,22 @@ sub decompose {
my $modifier_re = '^\s*'.$modifier_tag_re.'(' . join( '|', @{$pkg->modifiers}) . ')\b';
my $modifier_as_class_re = '^\s*(' . join( '|', @{$pkg->modifiers}) . '):\s*(\S+)';
- my $struct = $self->new_plan( level => $recursing );
+ my $struct = shift || $self->new_plan( level => $recursing );
+ $self->parse_tree( $struct ) if (!$self->parse_tree);
+
my $remainder = '';
my $last_type = '';
while (!$remainder) {
if (/^\s*$/) { # end of an explicit group
last;
+ } elsif (/$float_end_re/) { # end of an explicit group
+ warn "Encountered explicit float end\n" if $self->debug;
+
+ $remainder = $';
+ $_ = '';
+
+ $last_type = '';
} elsif (/$group_end_re/) { # end of an explicit group
warn "Encountered explicit group end\n" if $self->debug;
@@ -640,6 +670,15 @@ sub decompose {
}
$last_type = '';
+ } elsif (/$float_start_re/) { # start of an explicit float
+ warn "Encountered explicit float start\n" if $self->debug;
+
+ $self->floating_plan( $self->new_plan( floating => 1 ) ) if (!$self->floating_plan);
+ # pass the floating_plan struct to be modified by the float'ed chunk
+ my ($floating_plan, $subremainder) = $self->new->decompose( $', undef, undef, undef, $self->floating_plan);
+ $_ = $subremainder;
+
+ $last_type = '';
} elsif (/$group_start_re/) { # start of an explicit group
warn "Encountered explicit group start\n" if $self->debug;
@@ -655,12 +694,14 @@ sub decompose {
warn "Encountered AND\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( '('.$_.')', $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( $group_start.$_.$group_end, $current_class, $recursing + 1 );
$_ = $subremainder;
$struct = $self->new_plan( level => $recursing, joiner => '&' );
$struct->add_node($_) for ($LHS, $RHS);
+ $self->parse_tree( $struct ) if ($self->parse_tree == $LHS);
+
$last_type = 'AND';
} elsif (/$or_re/) { # ORed expression
$_ = $';
@@ -669,12 +710,14 @@ sub decompose {
warn "Encountered OR\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( '('.$_.')', $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( $group_start.$_.$group_end, $current_class, $recursing + 1 );
$_ = $subremainder;
$struct = $self->new_plan( level => $recursing, joiner => '|' );
$struct->add_node($_) for ($LHS, $RHS);
+ $self->parse_tree( $struct ) if ($self->parse_tree == $LHS);
+
$last_type = 'OR';
} elsif ($self->facet_class_count && /$facet_re/) { # changing current class
warn "Encountered facet: $1$2 => $3\n" if $self->debug;
@@ -742,7 +785,7 @@ sub decompose {
# $struct->joiner( '&' );
#
# $last_type = '';
- } elsif (/^\s*([^$group_end\s]+)/o) { # atom
+ } elsif (/^\s*([^$group_end\s]+)/o && /^\s*([^$float_end\s]+)/o) { # atom
warn "Encountered atom: $1\n" if $self->debug;
warn "Remainder: $'\n" if $self->debug;
@@ -910,6 +953,8 @@ sub abstract_query2str_impl {
my $qp_class ||= shift || 'QueryParser';
my $qpconfig = $QueryParser::parser_config{$qp_class};
+ my $fs = $qpconfig->{operators}{float_start};
+ my $fe = $qpconfig->{operators}{float_end};
my $gs = $qpconfig->{operators}{group_start};
my $ge = $qpconfig->{operators}{group_end};
my $and = $qpconfig->{operators}{and};
@@ -955,18 +1000,32 @@ sub abstract_query2str_impl {
}
if (exists $abstract_query->{children}) {
+
my $op = (keys(%{$abstract_query->{children}}))[0];
- $q .= join(
- " " . ($op eq '&' ? '' : $or) . " ",
- map {
- abstract_query2str_impl($_, $depth + 1, $qp_class)
- } @{$abstract_query->{children}{$op}}
- );
- $needs_group += scalar(@{$abstract_query->{children}{$op}});
+
+ if ($abstract_query->{floating}) { # always the top node!
+ my $sub_node = pop @{$abstract_query->{children}{$op}};
+
+ $abstract_query->{floating} = 0;
+ $q = $fs.abstract_query2str_impl($abstract_query,0,$qp_class).$fe;
+
+ $abstract_query = $sub_node;
+ }
+
+ if ($abstract_query && exists $abstract_query->{children}) {
+ $op = (keys(%{$abstract_query->{children}}))[0];
+ $q .= join(
+ " " . ($op eq '&' ? '' : $or) . " ",
+ map {
+ abstract_query2str_impl($_, $depth + 1, $qp_class)
+ } @{$abstract_query->{children}{$op}}
+ );
+ $needs_group += scalar(@{$abstract_query->{children}{$op}});
+ }
} elsif ($abstract_query->{'&'} or $abstract_query->{'|'}) {
my $op = (keys(%{$abstract_query}))[0];
$q .= join(
- " " . ($op eq '&' ? $and : $or) . " ",
+ " " . ($op eq '&' ? '' : $or) . " ",
map {
abstract_query2str_impl($_, $depth + 1, $qp_class)
} @{$abstract_query->{$op}}
@@ -1259,6 +1318,7 @@ sub to_abstract_query {
my $abstract_query = {
type => "query_plan",
+ floating => $self->{floating},
filters => [map { $_->to_abstract_query } @{$self->filters}],
modifiers => [map { $_->to_abstract_query } @{$self->modifiers}]
};
commit bdcfdfb259c33d11086e8732e1e689a60d2828cc
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Sep 7 15:51:43 2012 -0400
QP: OO-ize canonicalizer; remove extra nesting from canonicalized query; repair nested operator in bool nesting; updated (basis) test script
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/extras/fts-replacement.pl b/Open-ILS/src/extras/fts-replacement.pl
index 63148af..c18f2e8 100755
--- a/Open-ILS/src/extras/fts-replacement.pl
+++ b/Open-ILS/src/extras/fts-replacement.pl
@@ -35,7 +35,6 @@ GetOptions(
'runs=i' => \$runs
);
-print "Original query: $query\n";
my $start = time();
OpenILS::Application::Storage::Driver::Pg::QueryParser->new( superpage_size => $superpage_size, superpage => $superpage, core_limit => $core_limit, debug => $debug, query => $query )->parse->parse_tree for (1 .. $runs);
@@ -49,8 +48,10 @@ my $sql = $plan->toSQL;
$sql =~ s/^\s*$//gm;
print "SQL:\n$sql\n\n" if (!$quiet);
-my $abstract_query = $plan->parse_tree->to_abstract_query(with_config => 1);
+my $abstract_query = $plan->parse_tree->to_abstract_query(with_config => 0);
print "abstract_query: " . Dumper($abstract_query) . "\n";
+print "Original query: $query\n";
+print "Canonicalized query: ".$plan->canonicalize()."\n";
print "Simple plan: " . ($plan->simple_plan ? 'yes' : 'no') . "\n";
print "Total parse time, $runs runs: " . ($end - $start) . "s\n";
print "Average parse time, $runs runs: " . sprintf('%0.3f',(($end - $start) / $runs) * 1000) . "ms\n";
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 0c9f85c..3838dd3 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -19,6 +19,14 @@ our %parser_config = (
}
);
+sub canonicalize {
+ my $self = shift;
+ return QueryParser::Canonicalize::abstract_query2str_impl(
+ $self->parse_tree->to_abstract_query(@_)
+ );
+}
+
+
sub facet_class_count {
my $self = shift;
return @{$self->facet_classes};
@@ -647,7 +655,7 @@ sub decompose {
warn "Encountered AND\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( '('.$_.')', $current_class, $recursing + 1 );
$_ = $subremainder;
$struct = $self->new_plan( level => $recursing, joiner => '&' );
@@ -661,7 +669,7 @@ sub decompose {
warn "Encountered OR\n" if $self->debug;
my $LHS = $struct;
- my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 1 );
+ my ($RHS, $subremainder) = $self->decompose( '('.$_.')', $current_class, $recursing + 1 );
$_ = $subremainder;
$struct = $self->new_plan( level => $recursing, joiner => '|' );
@@ -836,12 +844,14 @@ sub compare_abstract_atoms {
}
sub fake_abstract_atom_from_phrase {
- my ($phrase, $neg) = @_;
+ my $phrase = shift;
+ my $neg = shift;
+ my $qp_class = shift || 'QueryParser';
my $prefix = '"';
if ($neg) {
$prefix =
- $QueryParser::parser_config{QueryParser}{operators}{disallowed} .
+ $QueryParser::parser_config{$qp_class}{operators}{disallowed} .
$prefix;
}
@@ -872,10 +882,11 @@ package QueryParser::Canonicalize; # not OO
sub _abstract_query2str_filter {
my $f = shift;
- my $qpconfig = $parser_config{QueryParser};
+ my $qp_class = shift || 'QueryParser';
+ my $qpconfig = $QueryParser::parser_config{$qp_class};
return sprintf(
- "%s%s(%s)",
+ '%s%s(%s)',
$f->{negate} ? $qpconfig->{operators}{disallowed} : "",
$f->{name},
join(",", @{$f->{args}})
@@ -884,7 +895,8 @@ sub _abstract_query2str_filter {
sub _abstract_query2str_modifier {
my $f = shift;
- my $qpconfig = $parser_config{QueryParser};
+ my $qp_class = shift || 'QueryParser';
+ my $qpconfig = $QueryParser::parser_config{$qp_class};
return $qpconfig->{operators}{modifier} . $f;
}
@@ -892,26 +904,31 @@ sub _abstract_query2str_modifier {
# This should produce an equivalent query to the original, given an
# abstract_query.
sub abstract_query2str_impl {
- my ($abstract_query, $depth) = @_;
+ my $abstract_query = shift;
+ my $depth = shift || 0;
- my $qpconfig = $parser_config{QueryParser};
+ my $qp_class ||= shift || 'QueryParser';
+ my $qpconfig = $QueryParser::parser_config{$qp_class};
my $gs = $qpconfig->{operators}{group_start};
my $ge = $qpconfig->{operators}{group_end};
my $and = $qpconfig->{operators}{and};
my $or = $qpconfig->{operators}{or};
+ my $needs_group = 0;
my $q = "";
- $q .= $gs if $abstract_query->{type} and $abstract_query->{type} eq "query_plan" and $depth;
if (exists $abstract_query->{type}) {
if ($abstract_query->{type} eq 'query_plan') {
- $q .= join(" ", map { _abstract_query2str_filter($_) } @{$abstract_query->{filters}}) if
+ $q .= join(" ", map { _abstract_query2str_filter($_, $qp_class) } @{$abstract_query->{filters}}) if
exists $abstract_query->{filters};
+ $needs_group += scalar(@{$abstract_query->{filters}}) if exists $abstract_query->{filters};
+
$q .= " ";
- $q .= join(" ", map { _abstract_query2str_modifier($_) } @{$abstract_query->{modifiers}}) if
+ $q .= join(" ", map { _abstract_query2str_modifier($_, $qp_class) } @{$abstract_query->{modifiers}}) if
exists $abstract_query->{modifiers};
+ $needs_group += scalar(@{$abstract_query->{modifiers}}) if exists $abstract_query->{modifiers};
} elsif ($abstract_query->{type} eq 'node') {
if ($abstract_query->{alias}) {
$q .= " " . $abstract_query->{alias};
@@ -927,34 +944,38 @@ sub abstract_query2str_impl {
$q .= $prefix .
($abstract_query->{content} || '') .
($abstract_query->{suffix} || '');
+ $needs_group += 1;
} elsif ($abstract_query->{type} eq 'facet') {
# facet syntax [ # ] is hardcoded I guess?
my $prefix = $abstract_query->{negate} ? $qpconfig->{operators}{disallowed} : '';
$q .= $prefix . $abstract_query->{name} . "[" .
join(" # ", @{$abstract_query->{values}}) . "]";
+ $needs_group += 1;
}
}
if (exists $abstract_query->{children}) {
my $op = (keys(%{$abstract_query->{children}}))[0];
$q .= join(
- " " . ($op eq '&' ? $and : $or) . " ",
+ " " . ($op eq '&' ? '' : $or) . " ",
map {
- abstract_query2str_impl($_, $depth + 1)
+ abstract_query2str_impl($_, $depth + 1, $qp_class)
} @{$abstract_query->{children}{$op}}
);
+ $needs_group += scalar(@{$abstract_query->{children}{$op}});
} elsif ($abstract_query->{'&'} or $abstract_query->{'|'}) {
my $op = (keys(%{$abstract_query}))[0];
$q .= join(
" " . ($op eq '&' ? $and : $or) . " ",
map {
- abstract_query2str_impl($_, $depth + 1)
+ abstract_query2str_impl($_, $depth + 1, $qp_class)
} @{$abstract_query->{$op}}
);
+ $needs_group += scalar(@{$abstract_query->{$op}});
}
$q .= " ";
- $q .= $ge if $abstract_query->{type} and $abstract_query->{type} eq "query_plan" and $depth;
+ $q = $gs . $q . $ge if ($needs_group > 1 and $depth);
return $q;
}
@@ -1530,7 +1551,7 @@ sub to_abstract_query {
last if $self->replace_phrase_in_abstract_query(
$tmplist,
$_,
- QueryParser::_util::fake_abstract_atom_from_phrase($phrase)
+ QueryParser::_util::fake_abstract_atom_from_phrase($phrase, undef, $pkg)
);
}
}
@@ -1563,7 +1584,7 @@ sub to_abstract_query {
last if $self->replace_phrase_in_abstract_query(
$tmplist,
$_,
- QueryParser::_util::fake_abstract_atom_from_phrase($phrase, 1)
+ QueryParser::_util::fake_abstract_atom_from_phrase($phrase, 1, $pkg)
);
}
}
commit 205ea5125eb0c22932ea5774d299d7cac2ba3301
Author: Mike Rylander <mrylander at gmail.com>
Date: Fri Sep 7 14:15:21 2012 -0400
Automatic push-down of explicitly-bool-connected conditions
Signed-off-by: Mike Rylander <mrylander at gmail.com>
Signed-off-by: Thomas Berezansky <tsbere at mvlc.org>
Signed-off-by: Lebbeous Fogle-Weekley <lebbeous at esilibrary.com>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
index 7ee0402..0c9f85c 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/QueryParser.pm
@@ -646,7 +646,12 @@ sub decompose {
next if ($last_type eq 'OR');
warn "Encountered AND\n" if $self->debug;
- $struct->joiner( '&' );
+ my $LHS = $struct;
+ my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 1 );
+ $_ = $subremainder;
+
+ $struct = $self->new_plan( level => $recursing, joiner => '&' );
+ $struct->add_node($_) for ($LHS, $RHS);
$last_type = 'AND';
} elsif (/$or_re/) { # ORed expression
@@ -655,7 +660,12 @@ sub decompose {
next if ($last_type eq 'OR');
warn "Encountered OR\n" if $self->debug;
- $struct->joiner( '|' );
+ my $LHS = $struct;
+ my ($RHS, $subremainder) = $self->decompose( $_, $current_class, $recursing + 1 );
+ $_ = $subremainder;
+
+ $struct = $self->new_plan( level => $recursing, joiner => '|' );
+ $struct->add_node($_) for ($LHS, $RHS);
$last_type = 'OR';
} elsif ($self->facet_class_count && /$facet_re/) { # changing current class
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/examples/fm_IDL.xml | 66 ++
Open-ILS/examples/opensrf.xml.example | 23 +-
Open-ILS/src/extras/fts-replacement.pl | 5 +-
.../lib/OpenILS/Application/Search/Biblio.pm | 186 ++---
.../Application/Storage/Driver/Pg/QueryParser.pm | 923 +++++++++++++++-----
.../Application/Storage/Publisher/metabib.pm | 202 +----
.../lib/OpenILS/Application/Storage/QueryParser.pm | 865 +++++++++++++++----
Open-ILS/src/perlmods/t/21-QueryParser.t | 307 +++++++
.../src/sql/Pg/000.english.pg90.fts-config.sql | 45 -
.../src/sql/Pg/000.english.pg91.fts-config.sql | 58 ++-
.../src/sql/Pg/000.english.pg92.fts-config.sql | 2 +-
Open-ILS/src/sql/Pg/002.schema.config.sql | 120 +--
Open-ILS/src/sql/Pg/030.schema.metabib.sql | 175 ++++
Open-ILS/src/sql/Pg/040.schema.asset.sql | 2 +-
Open-ILS/src/sql/Pg/300.schema.staged_search.sql | 327 -------
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 32 +
Open-ILS/src/sql/Pg/999.functions.global.sql | 21 +
.../sql/Pg/upgrade/0756.drop.query_parser_fts.sql | 9 +
.../src/sql/Pg/upgrade/0757.schema.ts_configs.sql | 434 +++++++++
.../support-scripts/test-scripts/query_tests.pl | 87 ++
.../conify/global/config/metabib_class.tt2 | 25 +
.../conify/global/config/metabib_class_ts_map.tt2 | 29 +
.../conify/global/config/metabib_field_ts_map.tt2 | 29 +
Open-ILS/web/opac/locale/en-US/lang.dtd | 3 +
.../xul/staff_client/chrome/content/main/menu.js | 12 +
.../chrome/content/main/menu_frame_menus.xul | 9 +
docs/QueryParser_Changes.txt | 47 +
docs/RELEASE_NOTES_NEXT/queryparser_changes.txt | 25 +
28 files changed, 2890 insertions(+), 1178 deletions(-)
create mode 100644 Open-ILS/src/perlmods/t/21-QueryParser.t
delete mode 100644 Open-ILS/src/sql/Pg/000.english.pg90.fts-config.sql
mode change 120000 => 100644 Open-ILS/src/sql/Pg/000.english.pg91.fts-config.sql
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0756.drop.query_parser_fts.sql
create mode 100644 Open-ILS/src/sql/Pg/upgrade/0757.schema.ts_configs.sql
create mode 100755 Open-ILS/src/support-scripts/test-scripts/query_tests.pl
create mode 100644 Open-ILS/src/templates/conify/global/config/metabib_class.tt2
create mode 100644 Open-ILS/src/templates/conify/global/config/metabib_class_ts_map.tt2
create mode 100644 Open-ILS/src/templates/conify/global/config/metabib_field_ts_map.tt2
create mode 100644 docs/QueryParser_Changes.txt
create mode 100644 docs/RELEASE_NOTES_NEXT/queryparser_changes.txt
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list