[open-ils-commits] r16067 - in trunk/Open-ILS/src: perlmods/OpenILS/Application/Search perlmods/OpenILS/Application/Storage/Publisher sql/Pg sql/Pg/upgrade (miker)
svn at svn.open-ils.org
svn at svn.open-ils.org
Wed Mar 31 03:43:51 EDT 2010
Author: miker
Date: 2010-03-31 03:43:46 -0400 (Wed, 31 Mar 2010)
New Revision: 16067
Added:
trunk/Open-ILS/src/sql/Pg/upgrade/0223.schema.staged_search.stored_proc.sql
Modified:
trunk/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm
trunk/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm
trunk/Open-ILS/src/sql/Pg/002.schema.config.sql
trunk/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
Log:
correct paging behavior with new staged search stored proc and results
Modified: trunk/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm 2010-03-30 20:48:02 UTC (rev 16066)
+++ trunk/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm 2010-03-31 07:43:46 UTC (rev 16067)
@@ -978,7 +978,7 @@
my $start = time;
$results = $U->storagereq($method, %$search_hash);
$search_duration = time - $start;
- $logger->info("staged search: DB call took $search_duration seconds");
+ $logger->info("staged search: DB call took $search_duration seconds and returned ".scalar(@$results)." rows, including summary");
$summary = shift(@$results);
unless($summary) {
@@ -1001,7 +1001,6 @@
}
push @$new_ids, grep {defined($_)} map {$_->[0]} @$results;
-
$results = [grep {defined $_->[0]} @$results];
cache_staged_search_page($key, $page, $summary, $results) if $docache;
}
Modified: trunk/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm
===================================================================
--- trunk/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm 2010-03-30 20:48:02 UTC (rev 16066)
+++ trunk/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm 2010-03-31 07:43:46 UTC (rev 16067)
@@ -2895,17 +2895,23 @@
# gather the limit or default to 10
- my $limit = $args{limit} || 10;
+ my $limit = $args{check_limit} || 'NULL';
if (my ($filter) = $query->parse_tree->find_filter('limit')) {
$limit = $filter->args->[0] if (@{$filter->args});
}
+ if (my ($filter) = $query->parse_tree->find_filter('check_limit')) {
+ $limit = $filter->args->[0] if (@{$filter->args});
+ }
# gather the offset or default to 0
- my $offset = $args{offset} || 0;
+ my $offset = $args{skip_check} || $args{offset} || 0;
if (my ($filter) = $query->parse_tree->find_filter('offset')) {
$offset = $filter->args->[0] if (@{$filter->args});
}
+ if (my ($filter) = $query->parse_tree->find_filter('skip_check')) {
+ $offset = $filter->args->[0] if (@{$filter->args});
+ }
# gather the estimation strategy or default to inclusion
@@ -2915,6 +2921,13 @@
}
+ # gather the estimation strategy or default to inclusion
+ my $core_limit = $args{core_limit};
+ if (my ($filter) = $query->parse_tree->find_filter('core_limit')) {
+ $core_limit = $filter->args->[0] if (@{$filter->args});
+ }
+
+
# gather statuses, and then forget those if we have an #available modifier
my @statuses;
if (my ($filter) = $query->parse_tree->find_filter('statuses')) {
@@ -2930,8 +2943,9 @@
}
- my $param_limit = $query->superpage_size || 'NULL';
- my $param_offset = 'NULL';
+ my $param_check = $limit || $query->superpage_size || 'NULL';
+ my $param_offset = $offset || 'NULL';
+ my $param_limit = $core_limit || 'NULL';
my $sp = $query->superpage || 1;
if ($sp > 1) {
@@ -2955,6 +2969,7 @@
$param_statuses\:\:INT[],
$param_locations\:\:INT[],
$param_offset\:\:INT,
+ $param_check\:\:INT,
$param_limit\:\:INT,
$staff\:\:BOOL,
$metarecord\:\:BOOL
@@ -2994,7 +3009,7 @@
$log->debug("Search yielded ".scalar(@$recs)." checked, visible results with an approximate visible total of $estimate.",DEBUG);
- for my $rec (@$recs[$offset .. $offset + $limit - 1]) {
+ for my $rec (@$recs) {
delete $$rec{checked};
delete $$rec{visible};
delete $$rec{excluded};
@@ -3082,6 +3097,9 @@
$query = "site($args{org_unit}) $query" if ($args{org_unit});
$query = "sort($args{sort}) $query" if ($args{sort});
$query = "limit($args{limit}) $query" if ($args{limit});
+ $query = "core_limit($args{core_limit}) $query" if ($args{core_limit});
+ $query = "skip_check($args{skip_check}) $query" if ($args{skip_check});
+ $query = "superpage($args{superpage}) $query" if ($args{superpage});
$query = "offset($args{offset}) $query" if ($args{offset});
$query = "#available $query" if ($args{available});
$query = "#descending $query" if ($args{sort_dir} && $args{sort_dir} =~ /^d/i);
Modified: trunk/Open-ILS/src/sql/Pg/002.schema.config.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-03-30 20:48:02 UTC (rev 16066)
+++ trunk/Open-ILS/src/sql/Pg/002.schema.config.sql 2010-03-31 07:43:46 UTC (rev 16067)
@@ -60,7 +60,7 @@
install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
-INSERT INTO config.upgrade_log (version) VALUES ('0222'); -- phasefx
+INSERT INTO config.upgrade_log (version) VALUES ('0223'); -- miker
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
Modified: trunk/Open-ILS/src/sql/Pg/300.schema.staged_search.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/300.schema.staged_search.sql 2010-03-30 20:48:02 UTC (rev 16066)
+++ trunk/Open-ILS/src/sql/Pg/300.schema.staged_search.sql 2010-03-31 07:43:46 UTC (rev 16067)
@@ -654,6 +654,7 @@
param_statuses INT[],
param_locations INT[],
param_offset INT,
+ param_check INT,
param_limit INT,
metarecord BOOL,
staff BOOL
@@ -664,6 +665,7 @@
current_res search.search_result%ROWTYPE;
search_org_list INT[];
+ check_limit INT;
core_limit INT;
core_offset INT;
tmp_int INT;
@@ -680,8 +682,9 @@
BEGIN
+ check_limit := COALESCE( param_check, 1000 );
+ core_limit := COALESCE( param_limit, 25000 );
core_offset := COALESCE( param_offset, 0 );
- core_limit := COALESCE( param_limit, 1000 );
-- core_skip_chk := COALESCE( param_skip_chk, 1 );
@@ -703,12 +706,13 @@
FETCH core_cursor INTO core_result;
EXIT WHEN NOT FOUND;
+ EXIT WHEN total_count >= core_limit;
total_count := total_count + 1;
- CONTINUE WHEN (total_count - 1 < core_offset);
+ CONTINUE WHEN total_count NOT BETWEEN core_offset + 1 AND check_limit + core_offset;
+
check_count := check_count + 1;
- CONTINUE WHEN (check_count > core_limit);
PERFORM 1 FROM biblio.record_entry b WHERE NOT b.deleted AND b.id IN ( SELECT * FROM search.explode_array( core_result.records ) );
IF NOT FOUND THEN
Added: trunk/Open-ILS/src/sql/Pg/upgrade/0223.schema.staged_search.stored_proc.sql
===================================================================
--- trunk/Open-ILS/src/sql/Pg/upgrade/0223.schema.staged_search.stored_proc.sql (rev 0)
+++ trunk/Open-ILS/src/sql/Pg/upgrade/0223.schema.staged_search.stored_proc.sql 2010-03-31 07:43:46 UTC (rev 16067)
@@ -0,0 +1,282 @@
+
+BEGIN;
+
+INSERT INTO config.upgrade_log (version) VALUES ('0223'); -- miker
+
+DROP FUNCTION search.query_parser_fts ( INT,INT,TEXT,INT[],INT[],INT,INT,BOOL,BOOL );
+
+CREATE OR REPLACE FUNCTION search.query_parser_fts (
+
+ param_search_ou INT,
+ param_depth INT,
+ param_query TEXT,
+ param_statuses INT[],
+ param_locations INT[],
+ param_offset INT,
+ param_check INT,
+ param_limit INT,
+ metarecord BOOL,
+ staff BOOL
+
+) RETURNS SETOF search.search_result AS $func$
+DECLARE
+
+ current_res search.search_result%ROWTYPE;
+ search_org_list INT[];
+
+ check_limit INT;
+ core_limit INT;
+ core_offset INT;
+ tmp_int INT;
+
+ core_result RECORD;
+ core_cursor REFCURSOR;
+ core_rel_query TEXT;
+
+ total_count INT := 0;
+ check_count INT := 0;
+ deleted_count INT := 0;
+ visible_count INT := 0;
+ excluded_count INT := 0;
+
+BEGIN
+
+ check_limit := COALESCE( param_check, 1000 );
+ core_limit := COALESCE( param_limit, 25000 );
+ core_offset := COALESCE( param_offset, 0 );
+
+ -- core_skip_chk := COALESCE( param_skip_chk, 1 );
+
+ IF param_search_ou > 0 THEN
+ IF param_depth IS NOT NULL THEN
+ SELECT array_accum(distinct id) INTO search_org_list FROM actor.org_unit_descendants( param_search_ou, param_depth );
+ ELSE
+ SELECT array_accum(distinct id) INTO search_org_list FROM actor.org_unit_descendants( param_search_ou );
+ END IF;
+ ELSIF param_search_ou < 0 THEN
+ SELECT array_accum(distinct org_unit) INTO search_org_list FROM actor.org_lasso_map WHERE lasso = -param_search_ou;
+ ELSIF param_search_ou = 0 THEN
+ -- reserved for user lassos (ou_buckets/type='lasso') with ID passed in depth ... hack? sure.
+ END IF;
+
+ OPEN core_cursor FOR EXECUTE param_query;
+
+ LOOP
+
+ FETCH core_cursor INTO core_result;
+ EXIT WHEN NOT FOUND;
+ EXIT WHEN total_count >= core_limit;
+
+ total_count := total_count + 1;
+
+ CONTINUE WHEN total_count NOT BETWEEN core_offset + 1 AND check_limit + core_offset;
+
+ check_count := check_count + 1;
+
+ PERFORM 1 FROM biblio.record_entry b WHERE NOT b.deleted AND b.id IN ( SELECT * FROM search.explode_array( core_result.records ) );
+ IF NOT FOUND THEN
+ -- RAISE NOTICE ' % were all deleted ... ', core_result.records;
+ deleted_count := deleted_count + 1;
+ CONTINUE;
+ END IF;
+
+ PERFORM 1
+ FROM biblio.record_entry b
+ JOIN config.bib_source s ON (b.source = s.id)
+ WHERE s.transcendant
+ AND b.id IN ( SELECT * FROM search.explode_array( core_result.records ) );
+
+ IF FOUND THEN
+ -- RAISE NOTICE ' % were all transcendant ... ', core_result.records;
+ visible_count := visible_count + 1;
+
+ current_res.id = core_result.id;
+ current_res.rel = core_result.rel;
+
+ tmp_int := 1;
+ IF metarecord THEN
+ SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
+ END IF;
+
+ IF tmp_int = 1 THEN
+ current_res.record = core_result.records[1];
+ ELSE
+ current_res.record = NULL;
+ END IF;
+
+ RETURN NEXT current_res;
+
+ CONTINUE;
+ END IF;
+
+ PERFORM 1
+ FROM asset.call_number cn
+ JOIN asset.uri_call_number_map map ON (map.call_number = cn.id)
+ JOIN asset.uri uri ON (map.uri = uri.id)
+ WHERE NOT cn.deleted
+ AND cn.label = '##URI##'
+ AND uri.active
+ AND ( param_locations IS NULL OR array_upper(param_locations, 1) IS NULL )
+ AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ AND cn.owning_lib IN ( SELECT * FROM search.explode_array( search_org_list ) )
+ LIMIT 1;
+
+ IF FOUND THEN
+ -- RAISE NOTICE ' % have at least one URI ... ', core_result.records;
+ visible_count := visible_count + 1;
+
+ current_res.id = core_result.id;
+ current_res.rel = core_result.rel;
+
+ tmp_int := 1;
+ IF metarecord THEN
+ SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
+ END IF;
+
+ IF tmp_int = 1 THEN
+ current_res.record = core_result.records[1];
+ ELSE
+ current_res.record = NULL;
+ END IF;
+
+ RETURN NEXT current_res;
+
+ CONTINUE;
+ END IF;
+
+ IF param_statuses IS NOT NULL AND array_upper(param_statuses, 1) > 0 THEN
+
+ PERFORM 1
+ FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ WHERE NOT cn.deleted
+ AND NOT cp.deleted
+ AND cp.status IN ( SELECT * FROM search.explode_array( param_statuses ) )
+ AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ AND cp.circ_lib IN ( SELECT * FROM search.explode_array( search_org_list ) )
+ LIMIT 1;
+
+ IF NOT FOUND THEN
+ -- RAISE NOTICE ' % were all status-excluded ... ', core_result.records;
+ excluded_count := excluded_count + 1;
+ CONTINUE;
+ END IF;
+
+ END IF;
+
+ IF param_locations IS NOT NULL AND array_upper(param_locations, 1) > 0 THEN
+
+ PERFORM 1
+ FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ WHERE NOT cn.deleted
+ AND NOT cp.deleted
+ AND cp.location IN ( SELECT * FROM search.explode_array( param_locations ) )
+ AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ AND cp.circ_lib IN ( SELECT * FROM search.explode_array( search_org_list ) )
+ LIMIT 1;
+
+ IF NOT FOUND THEN
+ -- RAISE NOTICE ' % were all copy_location-excluded ... ', core_result.records;
+ excluded_count := excluded_count + 1;
+ CONTINUE;
+ END IF;
+
+ END IF;
+
+ IF staff IS NULL OR NOT staff THEN
+
+ PERFORM 1
+ FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ JOIN actor.org_unit a ON (cp.circ_lib = a.id)
+ JOIN asset.copy_location cl ON (cp.location = cl.id)
+ JOIN config.copy_status cs ON (cp.status = cs.id)
+ WHERE NOT cn.deleted
+ AND NOT cp.deleted
+ AND cs.opac_visible
+ AND cl.opac_visible
+ AND cp.opac_visible
+ AND a.opac_visible
+ AND cp.circ_lib IN ( SELECT * FROM search.explode_array( search_org_list ) )
+ AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ LIMIT 1;
+
+ IF NOT FOUND THEN
+ -- RAISE NOTICE ' % were all visibility-excluded ... ', core_result.records;
+ excluded_count := excluded_count + 1;
+ CONTINUE;
+ END IF;
+
+ ELSE
+
+ PERFORM 1
+ FROM asset.call_number cn
+ JOIN asset.copy cp ON (cp.call_number = cn.id)
+ JOIN actor.org_unit a ON (cp.circ_lib = a.id)
+ JOIN asset.copy_location cl ON (cp.location = cl.id)
+ WHERE NOT cn.deleted
+ AND NOT cp.deleted
+ AND cp.circ_lib IN ( SELECT * FROM search.explode_array( search_org_list ) )
+ AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ LIMIT 1;
+
+ IF NOT FOUND THEN
+
+ PERFORM 1
+ FROM asset.call_number cn
+ WHERE cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) )
+ LIMIT 1;
+
+ IF FOUND THEN
+ -- RAISE NOTICE ' % were all visibility-excluded ... ', core_result.records;
+ excluded_count := excluded_count + 1;
+ CONTINUE;
+ END IF;
+
+ END IF;
+
+ END IF;
+
+ visible_count := visible_count + 1;
+
+ current_res.id = core_result.id;
+ current_res.rel = core_result.rel;
+
+ tmp_int := 1;
+ IF metarecord THEN
+ SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id;
+ END IF;
+
+ IF tmp_int = 1 THEN
+ current_res.record = core_result.records[1];
+ ELSE
+ current_res.record = NULL;
+ END IF;
+
+ RETURN NEXT current_res;
+
+ IF visible_count % 1000 = 0 THEN
+ -- RAISE NOTICE ' % visible so far ... ', visible_count;
+ END IF;
+
+ END LOOP;
+
+ current_res.id = NULL;
+ current_res.rel = NULL;
+ current_res.record = NULL;
+ current_res.total = total_count;
+ current_res.checked = check_count;
+ current_res.deleted = deleted_count;
+ current_res.visible = visible_count;
+ current_res.excluded = excluded_count;
+
+ CLOSE core_cursor;
+
+ RETURN NEXT current_res;
+
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;
+
More information about the open-ils-commits
mailing list