[open-ils-commits] r688 - in ESI-Examples/sys: . etc/syslog-ng scripts (dmcmorris)
svn at svn.open-ils.org
svn at svn.open-ils.org
Wed Sep 30 11:18:50 EDT 2009
Author: dmcmorris
Date: 2009-09-30 11:18:47 -0400 (Wed, 30 Sep 2009)
New Revision: 688
Added:
ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Client
ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Logger
ESI-Examples/sys/scripts/
ESI-Examples/sys/scripts/archive.sh
Log:
Adding multi-server syslog-ng config and a script to archive/remove old logs
Added: ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Client
===================================================================
--- ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Client (rev 0)
+++ ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Client 2009-09-30 15:18:47 UTC (rev 688)
@@ -0,0 +1,42 @@
+# syslog-ng.conf for multi-server environment (app server/client version)
+# Copyright (C) 2009 Equinox Software Inc. Don McMorris <dmcmorris at esilibrary.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+options {
+ chain_hostnames(off);
+ sync(0);
+ stats(43200);
+ time_reopen(10);
+};
+
+source src { unix-stream("/dev/log"); internal(); pipe("/proc/kmsg"); };
+
+destination messages { file("/var/log/messages"); };
+destination d_ext {
+#FIXME - set the below IP to the log server
+ tcp("192.168.1.10" port(514));
+ #udp("192.168.1.10" port(514));
+};
+
+filter f_system { not facility(local0, local1, local2, local3, local4, local5, local6, local7); };
+
+destination console_all { file("/dev/tty12"); };
+
+#log { source(src); filter(f_system); destination(messages); };
+log { source(src); destination(d_ext); };
+log { source(src); filter(f_system); destination(console_all); };
+
Added: ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Logger
===================================================================
--- ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Logger (rev 0)
+++ ESI-Examples/sys/etc/syslog-ng/syslog-ng.conf-MultiServer-Logger 2009-09-30 15:18:47 UTC (rev 688)
@@ -0,0 +1,149 @@
+# syslog-ng.conf for multi-server environment (central log server version)
+# Copyright (C) 2009 Equinox Software Inc. Don McMorris <dmcmorris at esilibrary.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+options {
+ chain_hostnames(off);
+ sync(10);
+ stats(3600);
+ owner(root);
+ group(opensrf);
+ dir_group(opensrf);
+ dir_perm(0750);
+ perm(0640);
+ create_dirs(yes);
+ time_reopen(10);
+ keep_hostname(yes);
+};
+
+
+# --------------------------------------------------------------------------------
+# Sources
+# --------------------------------------------------------------------------------
+
+# our log messages
+source s_internal { unix-stream("/dev/log"); internal(); pipe("/proc/kmsg"); };
+
+# incoming production log messages
+#FIXME - fix the IP to be right
+source s_external {
+ udp(ip(192.168.1.10) port(514));
+ tcp(ip(192.168.1.10) port(514) max-connections(100));
+};
+
+# --------------------------------------------------------------------------------
+# Destinations
+# --------------------------------------------------------------------------------
+
+# System message from all evergreen servers go here
+destination d_external_sys {
+ file("/var/log/evergreen/sys/$HOST/$YEAR/$MONTH/$DAY/$FACILITY.log"); };
+
+
+# Production logging -------------------------------------------------------
+destination d_external_osrfsys { # local0
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/osrfsys.$HOUR.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_osrfact { # local1
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/activity.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_transport { # local2
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/transport.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_postgres { # local3
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/pg.$HOUR.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_osrferr { # local0-3 LOG_ERROR
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/osrferror.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_pgerr { # local4 LOG_ERROR
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/pg_err.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_osrfwarn { # local0-3 LOG_WARN
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/osrfwarn.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_apache_gw { # local6
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/gateway.$HOUR.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_apache { # local7
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/ap_access.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+destination d_external_apache_err { # local6 + local7 LOG_ERROR
+ file("/var/log/evergreen/prod/$YEAR/$MONTH/$DAY/ap_error.log",
+ template("$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC $HOST $MSG\n") template-escape(no)); };
+
+
+# send this host's system logs here
+destination d_internal { file("/var/log/messages"); };
+
+# By default messages are logged to tty12...
+destination console_all { file("/dev/tty12"); };
+
+
+
+# --------------------------------------------------------------------------------
+# Filters
+# --------------------------------------------------------------------------------
+
+filter f_system { not facility(local0, local1, local2, local3, local4, local5, local6, local7); };
+filter f_osrf { facility(local0, local1, local2, local3); };
+filter f_apache { facility(local6, local7); };
+
+filter f_local0 { facility(local0); };
+filter f_local1 { facility(local1); };
+filter f_local2 { facility(local2); };
+filter f_local3 { facility(local3); };
+filter f_local4 { facility(local4); };
+filter f_local5 { facility(local5); };
+filter f_local6 { facility(local6); };
+filter f_local7 { facility(local7); };
+filter f_error { level(err..emerg); };
+filter f_warn { level(warn); };
+filter f_errwarn { level(warn..emerg); };
+
+
+# --------------------------------------------------------------------------------
+# Log Statements
+# --------------------------------------------------------------------------------
+
+
+# this host's system logs
+log { source(s_internal); destination(d_internal); };
+log { source(s_internal); destination(console_all); };
+
+# inbound system logs
+log { source(s_external); filter(f_system); destination(d_external_sys); };
+log { source(s_external_dev); filter(f_system); destination(d_external_sys); };
+log { source(s_external_demo); filter(f_system); destination(d_external_sys); };
+
+# --------------------------------------------------------------------------------------------------------
+# opensrf production logs
+log { source(s_external); filter(f_local0); destination(d_external_osrfsys); };
+log { source(s_external); filter(f_local1); destination(d_external_osrfact); };
+log { source(s_external); filter(f_local2); destination(d_external_transport); };
+log { source(s_external); filter(f_local3); destination(d_external_postgres); };
+log { source(s_external); filter(f_osrf); filter(f_error); destination(d_external_osrferr); };
+log { source(s_external); filter(f_osrf); filter(f_warn); destination(d_external_osrfwarn); };
+log { source(s_external); filter(f_local3); filter(f_error); destination(d_external_pgerr); };
+
+# apache logs
+log { source(s_external); filter(f_local6); destination(d_external_apache_gw); };
+log { source(s_external); filter(f_local7); destination(d_external_apache); };
+log { source(s_external); filter(f_apache); filter(f_errwarn); destination(d_external_apache_err); };
+#log { source(s_external); filter(f_apache); filter(f_error); destination(d_external_apache_err_all); };
+
+
Added: ESI-Examples/sys/scripts/archive.sh
===================================================================
--- ESI-Examples/sys/scripts/archive.sh (rev 0)
+++ ESI-Examples/sys/scripts/archive.sh 2009-09-30 15:18:47 UTC (rev 688)
@@ -0,0 +1,75 @@
+#!/bin/bash
+# Evergreen log file compression & cleanup script
+# Copyright (C) 2009 Equinox Software Inc. Don McMorris <dmcmorris at esilibrary.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+#
+# This script is intended to be run nightly. It will gzip the logfiles that are
+# in the log directory from $archivegoback days ago and remove files older than
+# $deletegoback days ago (excepting activity.log and ap_access.log)
+#
+# USAGE: archive.sh _BaseDir_ _ArchiveGoBackDays_ _DeleteGoBackDays_
+#
+# NOTE: This script assumes logfiles are stored in _BaseDir_/YYYY/MM/DD
+# as per example syslog-ng configs.
+
+basedir=$1
+archivegoback=$2
+deletegoback=$3
+
+# Make sure our command-line is complete...
+if [ -z $basedir ] || [ -z $archivegoback ] || [ -z $deletegoback ]; then
+ echo "USAGE: $0 _BaseDir_ _ArchiveGoBackDays_ _DeleteGoBackDays_"
+ exit 1
+fi
+
+# Make sure the base_dir exists...
+if [ ! -d $basedir ]; then
+ echo "$basedir does not exist!"
+ exit 1
+fi
+
+# figure out what dir we should be working with for archiving...
+archivemonth=$(date --date="$archivegoback day ago" +"%m");
+archiveyear=$(date --date="$archivegoback day ago" +"%Y");
+archiveday=$(date --date="$archivegoback day ago" +"%d");
+
+[ ${#archivemonth} == 1 ] && archivemonth="0$month";
+[ ${#archiveday} == 1 ] && archiveday="0$day";
+
+dir="$basedir/$archiveyear/$archivemonth/$archiveday";
+
+# Make sure $dir exists!
+if [ ! -d $dir ]; then
+ echo $dir does not exist!
+ exit 1
+fi
+
+# gzip the files in the directory from $archivegoback's directory
+cd $dir;
+echo "Archiving files in $dir";
+
+for i in `ls`; do
+ echo "Archiving $i";
+ gzip "$i";
+done;
+
+# remove files older than $deletegoback days (except ap_access.log and activity.log)
+echo "Deleting files in $basedir older than $deletegoback days"
+echo "(not including activity and ap_access logs)"
+find $basedir -not -name activity.* -not -name ap_access.* -type f -ctime +$deletegoback -exec rm {} \;
+echo "We're done!"
+exit 0
Property changes on: ESI-Examples/sys/scripts/archive.sh
___________________________________________________________________
Name: svn:executable
+ *
More information about the open-ils-commits
mailing list