[open-ils-commits] [GIT] Evergreen ILS branch master updated. 510d4febcbb666a6ca944470969e94e800f9de00
Evergreen Git
git at git.evergreen-ils.org
Fri May 26 12:14:22 EDT 2017
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".
The branch, master has been updated
via 510d4febcbb666a6ca944470969e94e800f9de00 (commit)
via 6ded4c35f463af9004651c9f73ac7f0036423f79 (commit)
via 36567bb7696e33f03ee1ab575d3ef6a78345ee0c (commit)
via ea536c71dedfaf9b97067e2da60e63a80be103fb (commit)
via 71817cb89fdc1cc782857293e7b87b3ea65e3378 (commit)
via 6214235b7ee8d041d1a4f06fd562281b30663db9 (commit)
from 234e1fa1db8434dd8c1866c001f6875d02eef907 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 510d4febcbb666a6ca944470969e94e800f9de00
Author: Bill Erickson <berickxx at gmail.com>
Date: Fri May 26 12:09:49 2017 -0400
LP#1672775 Stamping A/T purge SQL upgrade
Signed-off-by: Bill Erickson <berickxx at gmail.com>
diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index da3a39d..b19894a 100644
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -90,7 +90,7 @@ CREATE TRIGGER no_overlapping_deps
BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1042', :eg_version); -- mmorgan/gmcharlt
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('1044', :eg_version); -- berick/gmcharlt
CREATE TABLE config.bib_source (
id SERIAL PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql b/Open-ILS/src/sql/Pg/upgrade/1043.schema.action-trigger-purge-events.sql
similarity index 98%
rename from Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql
rename to Open-ILS/src/sql/Pg/upgrade/1043.schema.action-trigger-purge-events.sql
index 30f4dd6..6f0939b 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/1043.schema.action-trigger-purge-events.sql
@@ -1,6 +1,6 @@
BEGIN;
--- SELECT evergreen.upgrade_deps_block_check('1XXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('1043', :eg_version);
ALTER TABLE action_trigger.event_definition
ADD COLUMN retention_interval INTERVAL;
diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql b/Open-ILS/src/sql/Pg/upgrade/1044.data.action-trigger-purge-events.sql
similarity index 86%
rename from Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql
rename to Open-ILS/src/sql/Pg/upgrade/1044.data.action-trigger-purge-events.sql
index 1c79052..4dd3473 100644
--- a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/1044.data.action-trigger-purge-events.sql
@@ -1,6 +1,6 @@
BEGIN;
--- SELECT evergreen.upgrade_deps_block_check('1XXX', :eg_version);
+SELECT evergreen.upgrade_deps_block_check('1044', :eg_version);
UPDATE action_trigger.hook SET passive = FALSE WHERE key IN (
'format.po.html',
commit 6ded4c35f463af9004651c9f73ac7f0036423f79
Author: Galen Charlton <gmc at equinoxinitiative.org>
Date: Fri May 26 10:24:53 2017 -0400
LP#1672775: add purge_at_events.srfsh to example crontab
Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>
Signed-off-by: Bill Erickson <berickxx at gmail.com>
diff --git a/Open-ILS/examples/crontab.example b/Open-ILS/examples/crontab.example
index 7a9d9e7..8803b32 100644
--- a/Open-ILS/examples/crontab.example
+++ b/Open-ILS/examples/crontab.example
@@ -81,6 +81,9 @@ EG_BIN_DIR = /openils/bin
20 3 1 * * . ~/.bashrc && $EG_BIN_DIR/action_trigger_runner.pl --osrf-config $SRF_CORE --process-hooks --granularity monthly
25 3 1 1 * . ~/.bashrc && $EG_BIN_DIR/action_trigger_runner.pl --osrf-config $SRF_CORE --process-hooks --granularity yearly
+# Purge old A/T events
+15 1 * * * . ~/.bashrc && $EG_BIN_DIR/purge_at_events.srfsh
+
# Legacy overdue/predue notice generator. Change to suit.
#XML_FILE_PREFIX = /openils/var/data/overdue/overdue
#0 3 * * * . ~/.bashrc && cd $EG_BIN_DIR && ./generate_circ_notices.pl --osrf_config $SRF_CORE --notice-types overdue,predue --generate-global-templates --send-email > $XML_FILE_PREFIX.$(date +"\%F").xml
commit 36567bb7696e33f03ee1ab575d3ef6a78345ee0c
Author: Bill Erickson <berickxx at gmail.com>
Date: Thu Mar 16 12:33:55 2017 -0400
LP#1672775 Action/Trigger retention interval release notes
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>
diff --git a/docs/RELEASE_NOTES_NEXT/Administration/purge-at-events.adoc b/docs/RELEASE_NOTES_NEXT/Administration/purge-at-events.adoc
new file mode 100644
index 0000000..87e5fbe
--- /dev/null
+++ b/docs/RELEASE_NOTES_NEXT/Administration/purge-at-events.adoc
@@ -0,0 +1,63 @@
+Action/Trigger Events Data Purging
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Action/Trigger event definitions have a new field called "Retention
+Interval". When an optional interval value is applied, events and
+template output data linked to the event definition will be deleted
+from the database once they reach the specified age.
+
+Retention Interval Restrictions for Passive Hooks
++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Restrictions are placed on retention interval values for event definitions
+using passive hooks to prevent data from being deleted while it's still
+needed by the system.
+
+The presence of event data is how the system knows not to send duplicate
+events. As long as a scenario exists where a duplicate event may be
+generated, the events must be retained.
+
+To apply a retention interval value to a passive-hook event definition:
+
+ * The event definition must have a max_delay value.
+ * The retention interval must be larger than the difference between
+ the delay and max_delay values.
+
+For example, if the delay is 7 days and max_delay is 10 days, the retention
+interval must be greater than 3 days to ensure no duplicate events are
+created between the first event on day 7 and the end of the event validity
+window on day 10.
+
+Deployment
+++++++++++
+
+A new purge_at_events.sh script is installed in the bin directory
+(typically /openils/bin) wich should be added to CRON for regular
+maintanence.
+
+NOTE: On large data sets, this script can take a long time to run and
+create higher than normal I/O load as it churns though the event and
+event_output tables. You may wish to run the script by hand the first
+time so it can be monitored. It can be run in psql like so:
+
+[source,sql]
+---------------------------------------------------------------
+SELECT action_trigger.purge_events();
+---------------------------------------------------------------
+
+NOTE: On *very* large data sets (10s to 100s of millions of event and
+event_output rows), it may be advisable to first to repopulate the event
+and event_output tables with only the desired data before starting
+regular purges. This can be done, for example, using the copy to temp
+table, truncate source table, repopulate source table from temp table
+approach. This will be much faster than the purge_events() function
+in cases where most of the data will be purged.
+
+Hook Data Cleanup
++++++++++++++++++
+
+A number of action_trigger.hook entries which have always been treated
+as active hooks, though are configured as passive hooks, have been
+updated to properly reflect the non-passive-ness. This allows for
+simpler configuration of their retention interval values.
+
commit ea536c71dedfaf9b97067e2da60e63a80be103fb
Author: Bill Erickson <berickxx at gmail.com>
Date: Thu Mar 16 11:46:06 2017 -0400
LP#1672775 Action/Trigger CRON-able purge script
Installs a new purge_at_events.sh script into the Evergreen bin
directory. Invoked at regular intervals to remove expired events.
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>
diff --git a/Open-ILS/src/Makefile.am b/Open-ILS/src/Makefile.am
index 09a6439..253e8ab 100644
--- a/Open-ILS/src/Makefile.am
+++ b/Open-ILS/src/Makefile.am
@@ -67,6 +67,7 @@ core_scripts = $(examples)/oils_ctl.sh \
$(supportscr)/juv_to_adult.srfsh \
$(supportscr)/thaw_expired_frozen_holds.srfsh \
$(supportscr)/long-overdue-status-update.pl \
+ $(supportscr)/purge_at_events.srfsh \
$(supportscr)/purge_holds.srfsh \
$(supportscr)/purge_circulations.srfsh \
$(supportscr)/purge_pending_users.srfsh \
diff --git a/Open-ILS/src/support-scripts/purge_at_events.srfsh b/Open-ILS/src/support-scripts/purge_at_events.srfsh
new file mode 100755
index 0000000..f14b287
--- /dev/null
+++ b/Open-ILS/src/support-scripts/purge_at_events.srfsh
@@ -0,0 +1,7 @@
+#!/openils/bin/srfsh
+open open-ils.cstore
+request open-ils.cstore open-ils.cstore.transaction.begin
+request open-ils.cstore open-ils.cstore.json_query {"from":["action_trigger.purge_events"]}
+request open-ils.cstore open-ils.cstore.transaction.commit
+close open-ils.cstore
+
commit 71817cb89fdc1cc782857293e7b87b3ea65e3378
Author: Bill Erickson <berickxx at gmail.com>
Date: Thu Mar 16 10:53:43 2017 -0400
LP#1672775 Action/Trigger purge PGTAP tests
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>
diff --git a/Open-ILS/src/sql/Pg/live_t/purge-at-events.pg b/Open-ILS/src/sql/Pg/live_t/purge-at-events.pg
new file mode 100644
index 0000000..61b405b
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/live_t/purge-at-events.pg
@@ -0,0 +1,90 @@
+BEGIN;
+
+SELECT plan(8);
+
+-- event_definition 1 uses checkout.due (passive) hook
+\set evt_def_pasv 1
+
+-- event_definition 1 uses format.po.html (active) hook
+\set evt_def_actv 4
+
+/* ---------------------------------------------------------------
+ * Test Interval Constraints
+ * ---------------------------------------------------------------*/
+
+UPDATE action_trigger.event_definition
+ SET delay = '1 day', max_delay = NULL WHERE id = :evt_def_pasv;
+
+PREPARE delay_required AS
+ UPDATE action_trigger.event_definition
+ SET retention_interval = '1 hour' WHERE id = :evt_def_pasv;
+
+SELECT throws_ok('delay_required', 'P0001',
+ 'retention_interval requires max_delay', 'No Max Delay Test');
+
+UPDATE action_trigger.event_definition
+ SET max_delay = '2 days' WHERE id = :evt_def_pasv;
+
+PREPARE short_interval AS
+ UPDATE action_trigger.event_definition
+ SET retention_interval = '1 hour' WHERE id = :evt_def_pasv;
+
+SELECT throws_ok('short_interval', 'P0001',
+ 'retention_interval is too short', 'Short Interval Update Test');
+
+-- '2 days' is longer than " '2 days' - '1 day' "
+PREPARE ok_interval AS
+ UPDATE action_trigger.event_definition
+ SET retention_interval = '2 days' WHERE id = :evt_def_pasv;
+
+SELECT lives_ok('ok_interval', 'Long Interval Update Test');
+
+PREPARE any_interval AS
+ UPDATE action_trigger.event_definition
+ SET retention_interval = '1 min' WHERE id = :evt_def_actv;
+
+SELECT lives_ok('any_interval', 'Active hooks allow any interval');
+
+/* ---------------------------------------------------------------
+ * Test Purging
+ * ---------------------------------------------------------------*/
+
+INSERT INTO action_trigger.event_output (id, data, create_time)
+ VALUES
+ (1010, '', NOW()), -- passive, non-purged
+ (1011, '', NOW() - '4 days'::INTERVAL), -- passive purge
+ (1012, '', NOW() - '4 days'::INTERVAL), -- active purge
+ (1013, '', NOW()) -- orphan purge
+;
+
+INSERT INTO action_trigger.event (id, target, event_def, run_time,
+ start_time, update_time, state, template_output) VALUES
+ (1010, 1, :evt_def_pasv, NOW(), NOW(), NOW(), 'pending', NULL),
+ (1011, 1, :evt_def_pasv, NOW(), NOW(), NOW(), 'error', NULL),
+ (1012, 1, :evt_def_pasv, NOW(), NOW(), NOW() - '4 days'::INTERVAL, 'complete', 1011), -- purge
+ (1013, 1, :evt_def_pasv, NOW(), NOW(), NOW() - '4 days'::INTERVAL, 'pending', NULL),
+ (1014, 1, :evt_def_actv, NOW(), NOW(), NOW() - '10 minutes'::INTERVAL, 'complete', 1012), -- purge
+ (1015, 1, :evt_def_pasv, NOW(), NOW(), NOW(), 'complete', 1010)
+;
+
+SELECT action_trigger.purge_events();
+
+SELECT is( -- purged outputs
+ (SELECT COUNT(*) FROM action_trigger.event_output WHERE id IN (1011,1012,1013)),
+ 0::BIGINT, 'Expired Template Output Purged');
+
+SELECT is( -- non-purged outputs
+ (SELECT COUNT(*) FROM action_trigger.event_output WHERE id IN (1010)),
+ 1::BIGINT, 'Non-Expired Template Output Retained');
+
+SELECT is( -- purged events
+ (SELECT COUNT(*) FROM action_trigger.event WHERE id IN (1012,1014)),
+ 0::BIGINT, 'Expired Event Purged');
+
+SELECT is( -- non-purged events
+ (SELECT COUNT(*) FROM action_trigger.event WHERE id IN (1010,1011,1013,1015)),
+ 4::BIGINT, 'Non-Expired Event Retained');
+
+SELECT * FROM finish();
+ROLLBACK;
+
commit 6214235b7ee8d041d1a4f06fd562281b30663db9
Author: Bill Erickson <berickxx at gmail.com>
Date: Wed Mar 15 15:39:55 2017 -0400
LP#1672775 Action/Trigger retention interval SQL/IDL
Adds a new 'retention_interval' column for Action/Trigger Event
Definitions and an action_trigger.purge_events() function for deleting
events that have exceeded their configured retention time, including any
outputs linked to those events.
Signed-off-by: Bill Erickson <berickxx at gmail.com>
Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>
diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml
index 86a9f3b..b1d287e 100644
--- a/Open-ILS/examples/fm_IDL.xml
+++ b/Open-ILS/examples/fm_IDL.xml
@@ -1406,6 +1406,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
<field reporter:label="Message Library Path" name="message_library_path" reporter:datatype="text"/>
<field reporter:label="Environment Entries" name="env" oils_persist:virtual="true" reporter:datatype="link"/>
<field reporter:label="Parameters" name="params" oils_persist:virtual="true" reporter:datatype="link"/>
+ <field reporter:label="Retention Interval" name="retention_interval" reporter:datatype="interval"/>
</fields>
<links>
<link field="owner" reltype="has_a" key="id" map="" class="aou"/>
diff --git a/Open-ILS/src/sql/Pg/400.schema.action_trigger.sql b/Open-ILS/src/sql/Pg/400.schema.action_trigger.sql
index 81fce60..d9f292e 100644
--- a/Open-ILS/src/sql/Pg/400.schema.action_trigger.sql
+++ b/Open-ILS/src/sql/Pg/400.schema.action_trigger.sql
@@ -49,8 +49,8 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('pen
INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('penalty.PATRON_EXCEEDS_CHECKOUT_COUNT','ausp','Patron has exceeded allowed checkout count',TRUE);
INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('penalty.PATRON_EXCEEDS_COLLECTIONS_WARNING','ausp','Patron has exceeded maximum fine amount for collections department warning',TRUE);
INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('acqpo.activated','acqpo','Purchase order was activated',FALSE);
-INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('format.po.html','acqpo','Formats a Purchase Order as an HTML document',TRUE);
-INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES ('format.po.pdf','acqpo','Formats a Purchase Order as a PDF document',TRUE);
+INSERT INTO action_trigger.hook (key,core_type,description) VALUES ('format.po.html','acqpo','Formats a Purchase Order as an HTML document');
+INSERT INTO action_trigger.hook (key,core_type,description) VALUES ('format.po.pdf','acqpo','Formats a Purchase Order as a PDF document');
INSERT INTO action_trigger.hook (key,core_type,description) VALUES ('damaged','acp','Item marked damaged');
INSERT INTO action_trigger.hook (key,core_type,description) VALUES ('checkout.damaged','circ','A circulating item is marked damaged and the patron is fined');
INSERT INTO action_trigger.hook (key,core_type,description) VALUES ('renewal','circ','Item renewed to user');
@@ -195,11 +195,47 @@ CREATE TABLE action_trigger.event_definition (
message_usr_path TEXT,
message_library_path TEXT,
message_title TEXT,
+ retention_interval INTERVAL,
CONSTRAINT ev_def_owner_hook_val_react_clean_delay_once UNIQUE (owner, hook, validator, reactor, delay, delay_field),
CONSTRAINT ev_def_name_owner_once UNIQUE (owner, name)
);
+CREATE OR REPLACE FUNCTION action_trigger.check_valid_retention_interval()
+ RETURNS TRIGGER AS $_$
+BEGIN
+ /*
+ * 1. Retention intervals are always allowed on active hooks.
+ * 2. On passive hooks, retention intervals are only allowed
+ * when the event definition has a max_delay value and the
+ * retention_interval value is greater than the difference
+ * beteween the delay and max_delay values.
+ */
+ PERFORM TRUE FROM action_trigger.hook
+ WHERE key = NEW.hook AND NOT passive;
+
+ IF FOUND THEN
+ RETURN NEW;
+ END IF;
+
+ IF NEW.max_delay IS NOT NULL THEN
+ IF EXTRACT(EPOCH FROM NEW.retention_interval) >
+ ABS(EXTRACT(EPOCH FROM (NEW.max_delay - NEW.delay))) THEN
+ RETURN NEW; -- all good
+ ELSE
+ RAISE EXCEPTION 'retention_interval is too short';
+ END IF;
+ ELSE
+ RAISE EXCEPTION 'retention_interval requires max_delay';
+ END IF;
+END;
+$_$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER is_valid_retention_interval
+ BEFORE INSERT OR UPDATE ON action_trigger.event_definition
+ FOR EACH ROW WHEN (NEW.retention_interval IS NOT NULL)
+ EXECUTE PROCEDURE action_trigger.check_valid_retention_interval();
+
CREATE TABLE action_trigger.environment (
id SERIAL PRIMARY KEY,
event_def INT NOT NULL REFERENCES action_trigger.event_definition (id) DEFERRABLE INITIALLY DEFERRED,
@@ -246,5 +282,53 @@ CREATE TABLE action_trigger.event_params (
CONSTRAINT event_params_event_def_param_once UNIQUE (event_def,param)
);
+CREATE OR REPLACE FUNCTION action_trigger.purge_events() RETURNS VOID AS $_$
+/**
+ * Deleting expired events without simultaneously deleting their outputs
+ * creates orphaned outputs. Deleting their outputs and all of the events
+ * linking back to them, plus any outputs those events link to is messy and
+ * inefficient. It's simpler to handle them in 2 sweeping steps.
+ *
+ * 1. Delete expired events.
+ * 2. Delete orphaned event outputs.
+ *
+ * This has the added benefit of removing outputs that may have been
+ * orphaned by some other process. Such outputs are not usuable by
+ * the system.
+ *
+ * This does not guarantee that all events within an event group are
+ * purged at the same time. In such cases, the remaining events will
+ * be purged with the next instance of the purge (or soon thereafter).
+ * This is another nod toward efficiency over completeness of old
+ * data that's circling the bit bucket anyway.
+ */
+BEGIN
+
+ DELETE FROM action_trigger.event WHERE id IN (
+ SELECT evt.id
+ FROM action_trigger.event evt
+ JOIN action_trigger.event_definition def ON (def.id = evt.event_def)
+ WHERE def.retention_interval IS NOT NULL
+ AND evt.state <> 'pending'
+ AND evt.update_time < (NOW() - def.retention_interval)
+ );
+
+ WITH linked_outputs AS (
+ SELECT templates.id AS id FROM (
+ SELECT DISTINCT(template_output) AS id
+ FROM action_trigger.event WHERE template_output IS NOT NULL
+ UNION
+ SELECT DISTINCT(error_output) AS id
+ FROM action_trigger.event WHERE error_output IS NOT NULL
+ UNION
+ SELECT DISTINCT(async_output) AS id
+ FROM action_trigger.event WHERE async_output IS NOT NULL
+ ) templates
+ ) DELETE FROM action_trigger.event_output
+ WHERE id NOT IN (SELECT id FROM linked_outputs);
+
+END;
+$_$ LANGUAGE PLPGSQL;
+
COMMIT;
diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
index e3c34b4..9e0517f 100644
--- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql
+++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql
@@ -9441,7 +9441,7 @@ INSERT INTO action_trigger.event_params (event_def, param, value)
-- trigger data related to acq user requests
-INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
+INSERT INTO action_trigger.hook (key,core_type,description) VALUES (
'aur.ordered',
'aur',
oils_i18n_gettext(
@@ -9449,8 +9449,7 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
'A patron acquisition request has been marked On-Order.',
'ath',
'description'
- ),
- TRUE
+ )
), (
'aur.received',
'aur',
@@ -9459,8 +9458,7 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
'A patron acquisition request has been marked Received.',
'ath',
'description'
- ),
- TRUE
+ )
), (
'aur.cancelled',
'aur',
@@ -9469,8 +9467,7 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
'A patron acquisition request has been marked Cancelled.',
'ath',
'description'
- ),
- TRUE
+ )
), (
'aur.created',
'aur',
@@ -9479,8 +9476,7 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
'A patron has made an acquisitions request.',
'ath',
'description'
- ),
- TRUE
+ )
), (
'aur.rejected',
'aur',
@@ -9489,8 +9485,7 @@ INSERT INTO action_trigger.hook (key,core_type,description,passive) VALUES (
'A patron acquisition request has been rejected.',
'ath',
'description'
- ),
- TRUE
+ )
)
;
@@ -9764,12 +9759,11 @@ INSERT INTO action_trigger.environment ( event_def, path) VALUES
( 20, 'usr.home_ou' );
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.acqcle.html',
'acqcle',
- 'Formats claim events into a voucher',
- TRUE
+ 'Formats claim events into a voucher'
);
INSERT INTO action_trigger.event_definition (
@@ -9824,12 +9818,11 @@ INSERT INTO action_trigger.environment (event_def, path) VALUES
;
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.acqinv.html',
'acqinv',
- 'Formats invoices into a voucher',
- TRUE
+ 'Formats invoices into a voucher'
);
INSERT INTO action_trigger.event_definition (
@@ -10318,12 +10311,11 @@ INSERT INTO action_trigger.reactor (module,description) VALUES
-- self-check checkout receipt
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.selfcheck.checkout',
'circ',
- 'Formats circ objects for self-checkout receipt',
- TRUE
+ 'Formats circ objects for self-checkout receipt'
);
INSERT INTO action_trigger.event_definition (id, active, owner, name, hook, validator, reactor, group_field, granularity, template )
@@ -10424,12 +10416,11 @@ INSERT INTO action_trigger.environment ( event_def, path) VALUES
-- items out selfcheck receipt
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.selfcheck.items_out',
'circ',
- 'Formats items out for self-checkout receipt',
- TRUE
+ 'Formats items out for self-checkout receipt'
);
INSERT INTO action_trigger.event_definition (id, active, owner, name, hook, validator, reactor, group_field, granularity, template )
@@ -10472,12 +10463,11 @@ INSERT INTO action_trigger.environment ( event_def, path) VALUES
( 11, 'circ_lib.hours_of_operation'),
( 11, 'usr');
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.selfcheck.holds',
'ahr',
- 'Formats holds for self-checkout receipt',
- TRUE
+ 'Formats holds for self-checkout receipt'
);
INSERT INTO action_trigger.event_definition (id, active, owner, name, hook, validator, reactor, group_field, granularity, template )
@@ -10533,12 +10523,11 @@ INSERT INTO action_trigger.environment ( event_def, path) VALUES
-- fines receipt
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.selfcheck.fines',
'au',
- 'Formats fines for self-checkout receipt',
- TRUE
+ 'Formats fines for self-checkout receipt'
);
INSERT INTO action_trigger.event_definition (id, active, owner, name, hook, validator, reactor, granularity, template )
@@ -10583,12 +10572,11 @@ $$
$$
);
-INSERT INTO action_trigger.hook (key, core_type, description, passive)
+INSERT INTO action_trigger.hook (key, core_type, description)
VALUES (
'format.acqli.html',
'jub',
- 'Formats lineitem worksheet for titles received',
- TRUE
+ 'Formats lineitem worksheet for titles received'
);
INSERT INTO action_trigger.event_definition (id, active, owner, name, hook, validator, reactor, granularity, template)
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql
new file mode 100644
index 0000000..30f4dd6
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.action-trigger-purge-events.sql
@@ -0,0 +1,105 @@
+BEGIN;
+
+-- SELECT evergreen.upgrade_deps_block_check('1XXX', :eg_version);
+
+ALTER TABLE action_trigger.event_definition
+ ADD COLUMN retention_interval INTERVAL;
+
+CREATE OR REPLACE FUNCTION action_trigger.check_valid_retention_interval()
+ RETURNS TRIGGER AS $_$
+BEGIN
+
+ /*
+ * 1. Retention intervals are alwyas allowed on active hooks.
+ * 2. On passive hooks, retention intervals are only allowed
+ * when the event definition has a max_delay value and the
+ * retention_interval value is greater than the difference
+ * beteween the delay and max_delay values.
+ */
+ PERFORM TRUE FROM action_trigger.hook
+ WHERE key = NEW.hook AND NOT passive;
+
+ IF FOUND THEN
+ RETURN NEW;
+ END IF;
+
+ IF NEW.max_delay IS NOT NULL THEN
+ IF EXTRACT(EPOCH FROM NEW.retention_interval) >
+ ABS(EXTRACT(EPOCH FROM (NEW.max_delay - NEW.delay))) THEN
+ RETURN NEW; -- all good
+ ELSE
+ RAISE EXCEPTION 'retention_interval is too short';
+ END IF;
+ ELSE
+ RAISE EXCEPTION 'retention_interval requires max_delay';
+ END IF;
+END;
+$_$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER is_valid_retention_interval
+ BEFORE INSERT OR UPDATE ON action_trigger.event_definition
+ FOR EACH ROW WHEN (NEW.retention_interval IS NOT NULL)
+ EXECUTE PROCEDURE action_trigger.check_valid_retention_interval();
+
+CREATE OR REPLACE FUNCTION action_trigger.purge_events() RETURNS VOID AS $_$
+/**
+ * Deleting expired events without simultaneously deleting their outputs
+ * creates orphaned outputs. Deleting their outputs and all of the events
+ * linking back to them, plus any outputs those events link to is messy and
+ * inefficient. It's simpler to handle them in 2 sweeping steps.
+ *
+ * 1. Delete expired events.
+ * 2. Delete orphaned event outputs.
+ *
+ * This has the added benefit of removing outputs that may have been
+ * orphaned by some other process. Such outputs are not usuable by
+ * the system.
+ *
+ * This does not guarantee that all events within an event group are
+ * purged at the same time. In such cases, the remaining events will
+ * be purged with the next instance of the purge (or soon thereafter).
+ * This is another nod toward efficiency over completeness of old
+ * data that's circling the bit bucket anyway.
+ */
+BEGIN
+
+ DELETE FROM action_trigger.event WHERE id IN (
+ SELECT evt.id
+ FROM action_trigger.event evt
+ JOIN action_trigger.event_definition def ON (def.id = evt.event_def)
+ WHERE def.retention_interval IS NOT NULL
+ AND evt.state <> 'pending'
+ AND evt.update_time < (NOW() - def.retention_interval)
+ );
+
+ WITH linked_outputs AS (
+ SELECT templates.id AS id FROM (
+ SELECT DISTINCT(template_output) AS id
+ FROM action_trigger.event WHERE template_output IS NOT NULL
+ UNION
+ SELECT DISTINCT(error_output) AS id
+ FROM action_trigger.event WHERE error_output IS NOT NULL
+ UNION
+ SELECT DISTINCT(async_output) AS id
+ FROM action_trigger.event WHERE async_output IS NOT NULL
+ ) templates
+ ) DELETE FROM action_trigger.event_output
+ WHERE id NOT IN (SELECT id FROM linked_outputs);
+
+END;
+$_$ LANGUAGE PLPGSQL;
+
+
+/* -- UNDO --
+
+BEGIN;
+DROP FUNCTION IF EXISTS action_trigger.purge_events();
+DROP TRIGGER IF EXISTS is_valid_retention_interval ON action_trigger.event_definition;
+DROP FUNCTION IF EXISTS action_trigger.check_valid_retention_interval();
+ALTER TABLE action_trigger.event_definition DROP COLUMN retention_interval;
+COMMIT;
+
+*/
+
+COMMIT;
+
diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql
new file mode 100644
index 0000000..1c79052
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.action-trigger-purge-events.sql
@@ -0,0 +1,22 @@
+BEGIN;
+
+-- SELECT evergreen.upgrade_deps_block_check('1XXX', :eg_version);
+
+UPDATE action_trigger.hook SET passive = FALSE WHERE key IN (
+ 'format.po.html',
+ 'format.po.pdf',
+ 'format.selfcheck.checkout',
+ 'format.selfcheck.items_out',
+ 'format.selfcheck.holds',
+ 'format.selfcheck.fines',
+ 'format.acqcle.html',
+ 'format.acqinv.html',
+ 'format.acqli.html',
+ 'aur.ordered',
+ 'aur.received',
+ 'aur.cancelled',
+ 'aur.created',
+ 'aur.rejected'
+);
+
+COMMIT;
-----------------------------------------------------------------------
Summary of changes:
Open-ILS/examples/crontab.example | 3 +
Open-ILS/examples/fm_IDL.xml | 1 +
Open-ILS/src/Makefile.am | 1 +
Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +-
Open-ILS/src/sql/Pg/400.schema.action_trigger.sql | 88 ++++++++++++++++-
Open-ILS/src/sql/Pg/950.data.seed-values.sql | 52 ++++------
Open-ILS/src/sql/Pg/live_t/purge-at-events.pg | 90 +++++++++++++++++
.../1043.schema.action-trigger-purge-events.sql | 105 ++++++++++++++++++++
.../1044.data.action-trigger-purge-events.sql | 22 ++++
...ge_circulations.srfsh => purge_at_events.srfsh} | 2 +-
.../Administration/purge-at-events.adoc | 63 ++++++++++++
11 files changed, 393 insertions(+), 36 deletions(-)
create mode 100644 Open-ILS/src/sql/Pg/live_t/purge-at-events.pg
create mode 100644 Open-ILS/src/sql/Pg/upgrade/1043.schema.action-trigger-purge-events.sql
create mode 100644 Open-ILS/src/sql/Pg/upgrade/1044.data.action-trigger-purge-events.sql
copy Open-ILS/src/support-scripts/{purge_circulations.srfsh => purge_at_events.srfsh} (89%)
create mode 100644 docs/RELEASE_NOTES_NEXT/Administration/purge-at-events.adoc
hooks/post-receive
--
Evergreen ILS
More information about the open-ils-commits
mailing list