[open-ils-commits] r834 - in PINES-Examples/trunk: . scripts scripts/logtools (dmcmorris)
svn at svn.open-ils.org
svn at svn.open-ils.org
Tue Mar 16 16:44:11 EDT 2010
Author: dmcmorris
Date: 2010-03-16 16:44:05 -0400 (Tue, 16 Mar 2010)
New Revision: 834
Added:
PINES-Examples/trunk/scripts/
PINES-Examples/trunk/scripts/logtools/
PINES-Examples/trunk/scripts/logtools/JSON.pm
PINES-Examples/trunk/scripts/logtools/README
PINES-Examples/trunk/scripts/logtools/autovac-spikes.sh
PINES-Examples/trunk/scripts/logtools/checkin_count.sh
PINES-Examples/trunk/scripts/logtools/checkout_count.sh
PINES-Examples/trunk/scripts/logtools/ip_count.sh
PINES-Examples/trunk/scripts/logtools/method_by_hour.sh
PINES-Examples/trunk/scripts/logtools/method_count.sh
PINES-Examples/trunk/scripts/logtools/non_ws_logins.sh
PINES-Examples/trunk/scripts/logtools/null-requests.sh
PINES-Examples/trunk/scripts/logtools/ordered_queries.sh
PINES-Examples/trunk/scripts/logtools/pglong.sh
PINES-Examples/trunk/scripts/logtools/process.sh
PINES-Examples/trunk/scripts/logtools/req_count.sh
PINES-Examples/trunk/scripts/logtools/search_counts.sh
PINES-Examples/trunk/scripts/logtools/search_session.sh
PINES-Examples/trunk/scripts/logtools/searches.sh
PINES-Examples/trunk/scripts/logtools/tail-searches.sh
PINES-Examples/trunk/scripts/logtools/top_queries.sh
PINES-Examples/trunk/scripts/logtools/top_searches.sh
PINES-Examples/trunk/scripts/logtools/utils.sh
PINES-Examples/trunk/scripts/logtools/waits.sh
PINES-Examples/trunk/scripts/logtools/watch_searches.sh
PINES-Examples/trunk/scripts/logtools/ws_logins.sh
Log:
Adding several log analysis scripts
Added: PINES-Examples/trunk/scripts/logtools/JSON.pm
===================================================================
--- PINES-Examples/trunk/scripts/logtools/JSON.pm (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/JSON.pm 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,827 @@
+
+package JSON::number;
+sub new {
+ my $class = shift;
+ my $x = shift || $class;
+ return bless \$x => __PACKAGE__;
+}
+
+use overload ( '""' => \&toString );
+
+sub toString { defined($_[1]) ? ${$_[1]} : ${$_[0]} }
+
+package JSON::bool::true;
+sub new { return bless {} => __PACKAGE__ }
+use overload ( '""' => \&toString );
+use overload ( 'bool' => sub { 1 } );
+use overload ( '0+' => sub { 1 } );
+
+sub toString { 'true' }
+
+package JSON::bool::false;
+sub new { return bless {} => __PACKAGE__ }
+use overload ( '""' => \&toString );
+use overload ( 'bool' => sub { 0 } );
+use overload ( '0+' => sub { 0 } );
+
+sub toString { 'false' }
+
+package JSON;
+use Unicode::Normalize;
+use vars qw/%_class_map/;
+
+sub register_class_hint {
+ my $class = shift;
+ my %args = @_;
+
+ $_class_map{hints}{$args{hint}} = \%args;
+ $_class_map{classes}{$args{name}} = \%args;
+}
+
+sub _JSON_regex {
+ my $string = shift;
+
+ $string =~ s/^\s* (
+ { | # start object
+ \[ | # start array
+ -?\d+\.?\d* | # number literal
+ "(?:(?:\\[\"])|[^\"])*" | # string literal
+ (?:\/\*.+?\*\/) | # C comment
+ true | # bool true
+ false | # bool false
+ null | # undef()
+ : | # object key-value sep
+ , | # list sep
+ \] | # array end
+ } # object end
+ )
+ \s*//sox;
+ return ($string,$1);
+}
+
+sub lookup_class {
+ my $self = shift;
+ my $hint = shift;
+ return $_class_map{hints}{$hint}{name}
+}
+
+sub lookup_hint {
+ my $self = shift;
+ my $class = shift;
+ return $_class_map{classes}{$class}{hint}
+}
+
+sub _json_hint_to_class {
+ my $type = shift;
+ my $hint = shift;
+
+ return $_class_map{hints}{$hint}{name} if (exists $_class_map{hints}{$hint});
+
+ $type = 'hash' if ($type eq '}');
+ $type = 'array' if ($type eq ']');
+
+ JSON->register_class_hint(name => $hint, hint => $hint, type => $type);
+
+ return $hint;
+}
+
+sub JSON2perl {
+ my $class = shift;
+ local $_ = shift;
+
+ s/(?<!\\)\$/\\\$/gmo; # fixup $ for later
+ s/(?<!\\)\@/\\\@/gmo; # fixup @ for later
+ s/(?<!\\)\%/\\\%/gmo; # fixup % for later
+
+ # Convert JSON Unicode...
+ s/\\u([0-9a-fA-F]{4})/chr(hex($1))/esog;
+
+ # handle class blessings
+ s/\/\*--\s*S\w*?\s+\S+\s*--\*\// bless(/sog;
+ s/(\]|\}|")\s*\/\*--\s*E\w*?\s+(\S+)\s*--\*\//$1 => _json_hint_to_class("$1", "$2")) /sog;
+
+ my $re = qr/((?<!\\)"(?>(?<=\\)"|[^"])*(?<!\\)")/;
+ # Grab strings...
+ my @strings = /$re/sog;
+
+ # Replace with code...
+ #s/"(?:(?:\\[\"])|[^\"])*"/ do{ \$t = '"'.shift(\@strings).'"'; eval \$t;} /sog;
+ s/$re/ eval shift(\@strings) /sog;
+
+ # Perlify hash notation
+ s/:/ => /sog;
+
+ # Do numbers...
+ #s/\b(-?\d+\.?\d*)\b/ JSON::number::new($1) /sog;
+
+ # Change javascript stuff to perl...
+ s/null/ undef /sog;
+ s/true/ bless( {}, "JSON::bool::true") /sog;
+ s/false/ bless( {}, "JSON::bool::false") /sog;
+
+ my $ret;
+ return eval '$ret = '.$_;
+}
+
+my $_json_index;
+sub ___JSON2perl {
+ my $class = shift;
+ my $data = shift;
+
+ $data = [ split //, $data ];
+
+ $_json_index = 0;
+
+ return _json_parse_data($data);
+}
+
+sub _eat_WS {
+ my $data = shift;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+}
+
+sub _json_parse_data {
+ my $data = shift;
+
+ my $out;
+
+ #warn "parse_data";
+
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ my $class = '';
+
+ my $c = $$data[$_json_index];
+
+ if ($c eq '/') {
+ $_json_index++;
+ $class = _json_parse_comment($data);
+
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ $c = $$data[$_json_index];
+ }
+
+ if ($c eq '"') {
+ $_json_index++;
+ my $val = '';
+
+ my $seen_slash = 0;
+ my $done = 0;
+ while (!$done) {
+ my $c = $$data[$_json_index];
+ #warn "c is $c";
+
+ if ($c eq '\\') {
+ if ($seen_slash) {
+ $val .= '\\';
+ $seen_slash = 0;
+ } else {
+ $seen_slash = 1;
+ }
+ } elsif ($c eq '"') {
+ if ($seen_slash) {
+ $val .= '"';
+ $seen_slash = 0;
+ } else {
+ $done = 1;
+ }
+ } elsif ($c eq 't') {
+ if ($seen_slash) {
+ $val .= "\t";
+ $seen_slash = 0;
+ } else {
+ $val .= 't';
+ }
+ } elsif ($c eq 'b') {
+ if ($seen_slash) {
+ $val .= "\b";
+ $seen_slash = 0;
+ } else {
+ $val .= 'b';
+ }
+ } elsif ($c eq 'f') {
+ if ($seen_slash) {
+ $val .= "\f";
+ $seen_slash = 0;
+ } else {
+ $val .= 'f';
+ }
+ } elsif ($c eq 'r') {
+ if ($seen_slash) {
+ $val .= "\r";
+ $seen_slash = 0;
+ } else {
+ $val .= 'r';
+ }
+ } elsif ($c eq 'n') {
+ if ($seen_slash) {
+ $val .= "\n";
+ $seen_slash = 0;
+ } else {
+ $val .= 'n';
+ }
+ } elsif ($c eq 'u') {
+ if ($seen_slash) {
+ $_json_index++;
+ $val .= chr(hex(join('',$$data[$_json_index .. $_json_index + 3])));
+ $_json_index += 3;
+ $seen_slash = 0;
+ } else {
+ $val .= 'u';
+ }
+ } else {
+ $val .= $c;
+ }
+ $_json_index++;
+
+ #warn "string is $val";
+ }
+
+ $out = $val;
+
+ #$out = _json_parse_string($data);
+ } elsif ($c eq '[') {
+ $_json_index++;
+ $out = [];
+
+ my $in_parse = 0;
+ my $done = 0;
+ while(!$done) {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] eq ']') {
+ $done = 1;
+ $_json_index++;
+ last;
+ }
+
+ if ($in_parse) {
+ if ($$data[$_json_index] ne ',') {
+ #warn "_json_parse_array: bad data, leaving array parser";
+ last;
+ }
+ $_json_index++;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ }
+
+ my $item = _json_parse_data($data);
+
+ push @$out, $item;
+ $in_parse++;
+ }
+
+ #$out = _json_parse_array($data);
+ } elsif ($c eq '{') {
+ $_json_index++;
+ $out = {};
+
+ my $in_parse = 0;
+ my $done = 0;
+ while(!$done) {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] eq '}') {
+ $done = 1;
+ $_json_index++;
+ last;
+ }
+
+ if ($in_parse) {
+ if ($$data[$_json_index] ne ',') {
+ #warn "_json_parse_object: bad data, leaving object parser";
+ last;
+ }
+ $_json_index++;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ }
+
+ my ($key,$value);
+ $key = _json_parse_data($data);
+
+ #warn "object key is $key";
+
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] ne ':') {
+ #warn "_json_parse_object: bad data, leaving object parser";
+ last;
+ }
+ $_json_index++;
+ $value = _json_parse_data($data);
+
+ $out->{$key} = $value;
+ $in_parse++;
+ }
+ #$out = _json_parse_object($data);
+ } elsif (lc($c) eq 'n') {
+ if (lc(join('',$$data[$_json_index .. $_json_index + 3])) eq 'null') {
+ $_json_index += 4;
+ } else {
+ warn "CRAP! bad null parsing...";
+ }
+ $out = undef;
+ #$out = _json_parse_null($data);
+ } elsif (lc($c) eq 't' or lc($c) eq 'f') {
+ if (lc(join('',$$data[$_json_index .. $_json_index + 3])) eq 'true') {
+ $out = 1;
+ $_json_index += 4;
+ } elsif (lc(join('',$$data[$_json_index .. $_json_index + 4])) eq 'false') {
+ $out = 0;
+ $_json_index += 5;
+ } else {
+ #warn "CRAP! bad bool parsing...";
+ $out = undef;
+ }
+ #$out = _json_parse_bool($data);
+ } elsif ($c =~ /\d+/o or $c eq '.' or $c eq '-') {
+ my $val;
+ while ($$data[$_json_index] =~ /[-\.0-9]+/io) {
+ $val .= $$data[$_json_index];
+ $_json_index++;
+ }
+ $out = 0+$val;
+ #$out = _json_parse_number($data);
+ }
+
+ if ($class) {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ my $c = $$data[$_json_index];
+
+ if ($c eq '/') {
+ $_json_index++;
+ _json_parse_comment($data)
+ }
+
+ bless( $out => lookup_class($class) );
+ }
+
+ $out;
+}
+
+sub _json_parse_null {
+ my $data = shift;
+
+ #warn "parse_null";
+
+ if (lc(join('',$$data[$_json_index .. $_json_index + 3])) eq 'null') {
+ $_json_index += 4;
+ } else {
+ #warn "CRAP! bad null parsing...";
+ }
+ return undef;
+}
+
+sub _json_parse_bool {
+ my $data = shift;
+
+ my $out;
+
+ #warn "parse_bool";
+
+ if (lc(join('',$$data[$_json_index .. $_json_index + 3])) eq 'true') {
+ $out = 1;
+ $_json_index += 4;
+ } elsif (lc(join('',$$data[$_json_index .. $_json_index + 4])) eq 'false') {
+ $out = 0;
+ $_json_index += 5;
+ } else {
+ #warn "CRAP! bad bool parsing...";
+ $out = undef;
+ }
+ return $out;
+}
+
+sub _json_parse_number {
+ my $data = shift;
+
+ #warn "parse_number";
+
+ my $val;
+ while ($$data[$_json_index] =~ /[-\.0-9]+/io) {
+ $val .= $$data[$_json_index];
+ $_json_index++;
+ }
+
+ return 0+$val;
+}
+
+sub _json_parse_object {
+ my $data = shift;
+
+ #warn "parse_object";
+
+ my $out = {};
+
+ my $in_parse = 0;
+ my $done = 0;
+ while(!$done) {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] eq '}') {
+ $done = 1;
+ $_json_index++;
+ last;
+ }
+
+ if ($in_parse) {
+ if ($$data[$_json_index] ne ',') {
+ #warn "_json_parse_object: bad data, leaving object parser";
+ last;
+ }
+ $_json_index++;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ }
+
+ my ($key,$value);
+ $key = _json_parse_data($data);
+
+ #warn "object key is $key";
+
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] ne ':') {
+ #warn "_json_parse_object: bad data, leaving object parser";
+ last;
+ }
+ $_json_index++;
+ $value = _json_parse_data($data);
+
+ $out->{$key} = $value;
+ $in_parse++;
+ }
+
+ return $out;
+}
+
+sub _json_parse_array {
+ my $data = shift;
+
+ #warn "parse_array";
+
+ my $out = [];
+
+ my $in_parse = 0;
+ my $done = 0;
+ while(!$done) {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+
+ if ($$data[$_json_index] eq ']') {
+ $done = 1;
+ $_json_index++;
+ last;
+ }
+
+ if ($in_parse) {
+ if ($$data[$_json_index] ne ',') {
+ #warn "_json_parse_array: bad data, leaving array parser";
+ last;
+ }
+ $_json_index++;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ }
+
+ my $item = _json_parse_data($data);
+
+ push @$out, $item;
+ $in_parse++;
+ }
+
+ return $out;
+}
+
+
+sub _json_parse_string {
+ my $data = shift;
+
+ #warn "parse_string";
+
+ my $val = '';
+
+ my $seen_slash = 0;
+ my $done = 0;
+ while (!$done) {
+ my $c = $$data[$_json_index];
+ #warn "c is $c";
+
+ if ($c eq '\\') {
+ if ($seen_slash) {
+ $val .= '\\';
+ $seen_slash = 0;
+ } else {
+ $seen_slash = 1;
+ }
+ } elsif ($c eq '"') {
+ if ($seen_slash) {
+ $val .= '"';
+ $seen_slash = 0;
+ } else {
+ $done = 1;
+ }
+ } elsif ($c eq 't') {
+ if ($seen_slash) {
+ $val .= "\t";
+ $seen_slash = 0;
+ } else {
+ $val .= 't';
+ }
+ } elsif ($c eq 'b') {
+ if ($seen_slash) {
+ $val .= "\b";
+ $seen_slash = 0;
+ } else {
+ $val .= 'b';
+ }
+ } elsif ($c eq 'f') {
+ if ($seen_slash) {
+ $val .= "\f";
+ $seen_slash = 0;
+ } else {
+ $val .= 'f';
+ }
+ } elsif ($c eq 'r') {
+ if ($seen_slash) {
+ $val .= "\r";
+ $seen_slash = 0;
+ } else {
+ $val .= 'r';
+ }
+ } elsif ($c eq 'n') {
+ if ($seen_slash) {
+ $val .= "\n";
+ $seen_slash = 0;
+ } else {
+ $val .= 'n';
+ }
+ } elsif ($c eq 'u') {
+ if ($seen_slash) {
+ $_json_index++;
+ $val .= chr(hex(join('',$$data[$_json_index .. $_json_index + 3])));
+ $_json_index += 3;
+ $seen_slash = 0;
+ } else {
+ $val .= 'u';
+ }
+ } else {
+ $val .= $c;
+ }
+ $_json_index++;
+
+ #warn "string is $val";
+ }
+
+ return $val;
+}
+
+sub _json_parse_comment {
+ my $data = shift;
+
+ #warn "parse_comment";
+
+ if ($$data[$_json_index] eq '/') {
+ $_json_index++;
+ while (!($$data[$_json_index] eq "\n")) { $_json_index++ }
+ $_json_index++;
+ return undef;
+ }
+
+ my $class = '';
+
+ if (join('',$$data[$_json_index .. $_json_index + 2]) eq '*--') {
+ $_json_index += 3;
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ if ($$data[$_json_index] eq 'S') {
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ while ($$data[$_json_index] !~ /[-\s]+/o) {
+ $class .= $$data[$_json_index];
+ $_json_index++;
+ }
+ while ($$data[$_json_index] =~ /\s+/o) { $_json_index++ }
+ }
+ }
+
+ while ($$data[$_json_index] ne '/') { $_json_index++ };
+ $_json_index++;
+
+ return $class;
+}
+
+sub old_JSON2perl {
+ my ($class, $json) = @_;
+
+ if (!defined($json)) {
+ return undef;
+ }
+
+ $json =~ s/(?<!\\)\$/\\\$/gmo; # fixup $ for later
+ $json =~ s/(?<!\\)\@/\\\@/gmo; # fixup @ for later
+ $json =~ s/(?<!\\)\%/\\\%/gmo; # fixup % for later
+
+ my @casts;
+ my $casting_depth = 0;
+ my $current_cast;
+ my $element;
+ my $output = '';
+ while (($json,$element) = _JSON_regex($json)) {
+
+ last unless ($element);
+
+ if ($element eq 'null') {
+ $output .= ' undef() ';
+ next;
+ } elsif ($element =~ /^\/\*--\s*S\w*?\s+(\w+)\s*--\*\/$/) {
+ my $hint = $1;
+ if (exists $_class_map{hints}{$hint}) {
+ $casts[$casting_depth] = $hint;
+ $output .= ' bless(';
+ }
+ next;
+ } elsif ($element =~ /^\/\*/) {
+ next;
+ } elsif ($element =~ /^\d/) {
+ $output .= "do { JSON::number::new($element) }";
+ next;
+ } elsif ($element eq '{' or $element eq '[') {
+ $casting_depth++;
+ } elsif ($element eq '}' or $element eq ']') {
+ $casting_depth--;
+ my $hint = $casts[$casting_depth];
+ $casts[$casting_depth] = undef;
+ if (defined $hint and exists $_class_map{hints}{$hint}) {
+ $output .= $element . ',"'. $_class_map{hints}{$hint}{name} . '")';
+ next;
+ }
+ } elsif ($element eq ':') {
+ $output .= ' => ';
+ next;
+ } elsif ($element eq 'true') {
+ $output .= 'bless( {}, "JSON::bool::true")';
+ next;
+ } elsif ($element eq 'false') {
+ $output .= 'bless( {}, "JSON::bool::false")';
+ next;
+ }
+
+ $output .= $element;
+ }
+
+ return eval $output;
+}
+
+sub perl2JSON {
+ my ($class, $perl, $strict) = @_;
+
+ my $output = '';
+ if (!defined($perl)) {
+ $output = '' if $strict;
+ $output = 'null' unless $strict;
+ } elsif (ref($perl) and ref($perl) =~ /^JSON/) {
+ $output .= $perl;
+ } elsif ( ref($perl) && exists($_class_map{classes}{ref($perl)}) ) {
+ $output .= '/*--S '.$_class_map{classes}{ref($perl)}{hint}.'--*/';
+ if (lc($_class_map{classes}{ref($perl)}{type}) eq 'hash') {
+ my %hash = %$perl;
+ $output .= perl2JSON(undef,\%hash, $strict);
+ } elsif (lc($_class_map{classes}{ref($perl)}{type}) eq 'array') {
+ my @array = @$perl;
+ $output .= perl2JSON(undef,\@array, $strict);
+ }
+ $output .= '/*--E '.$_class_map{classes}{ref($perl)}{hint}.'--*/';
+ } elsif (ref($perl) and ref($perl) =~ /HASH/) {
+ $output .= '{';
+ my $c = 0;
+ for my $key (sort keys %$perl) {
+ my $outkey = NFC($key);
+ $output .= ',' if ($c);
+
+ $outkey =~ s{\\}{\\\\}sgo;
+ $outkey =~ s/"/\\"/sgo;
+ $outkey =~ s/\t/\\t/sgo;
+ $outkey =~ s/\f/\\f/sgo;
+ $outkey =~ s/\r/\\r/sgo;
+ $outkey =~ s/\n/\\n/sgo;
+ $outkey =~ s/([\x{0080}-\x{fffd}])/sprintf('\u%0.4x',ord($1))/sgoe;
+
+ $output .= '"'.$outkey.'":'. perl2JSON(undef,$$perl{$key}, $strict);
+ $c++;
+ }
+ $output .= '}';
+ } elsif (ref($perl) and ref($perl) =~ /ARRAY/) {
+ $output .= '[';
+ my $c = 0;
+ for my $part (@$perl) {
+ $output .= ',' if ($c);
+
+ $output .= perl2JSON(undef,$part, $strict);
+ $c++;
+ }
+ $output .= ']';
+ } elsif (ref($perl) and ref($perl) =~ /CODE/) {
+ $output .= perl2JSON(undef,$perl->(), $strict);
+ } elsif (ref($perl) and ("$perl" =~ /^([^=]+)=(\w+)/o)) {
+ my $type = $2;
+ my $name = $1;
+ JSON->register_class_hint(name => $name, hint => $name, type => lc($type));
+ $output .= perl2JSON(undef,$perl, $strict);
+ } else {
+ $perl = NFC($perl);
+ $perl =~ s{\\}{\\\\}sgo;
+ $perl =~ s/"/\\"/sgo;
+ $perl =~ s/\t/\\t/sgo;
+ $perl =~ s/\f/\\f/sgo;
+ $perl =~ s/\r/\\r/sgo;
+ $perl =~ s/\n/\\n/sgo;
+ $perl =~ s/([\x{0080}-\x{fffd}])/sprintf('\u%0.4x',ord($1))/sgoe;
+ if (length($perl) < 10 and $perl =~ /^(?:\+|-)?\d*\.?\d+$/o and $perl !~ /^(?:\+|-)?0\d+/o ) {
+ $output = $perl;
+ } else {
+ $output = '"'.$perl.'"';
+ }
+ }
+
+ return $output;
+}
+
+my $depth = 0;
+sub perl2prettyJSON {
+ my ($class, $perl, $nospace) = @_;
+ $perl ||= $class;
+
+ my $output = '';
+ if (!defined($perl)) {
+ $output = " "x$depth unless($nospace);
+ $output .= 'null';
+ } elsif (ref($perl) and ref($perl) =~ /^JSON/) {
+ $output = " "x$depth unless($nospace);
+ $output .= $perl;
+ } elsif ( ref($perl) && exists($_class_map{classes}{ref($perl)}) ) {
+ $depth++;
+ $output .= "\n";
+ $output .= " "x$depth;
+ $output .= '/*--S '.$_class_map{classes}{ref($perl)}{hint}."--*/ ";
+ if (lc($_class_map{classes}{ref($perl)}{type}) eq 'hash') {
+ my %hash = %$perl;
+ $output .= perl2prettyJSON(\%hash,undef,1);
+ } elsif (lc($_class_map{classes}{ref($perl)}{type}) eq 'array') {
+ my @array = @$perl;
+ $output .= perl2prettyJSON(\@array,undef,1);
+ }
+ $output .= ' /*--E '.$_class_map{classes}{ref($perl)}{hint}.'--*/';
+ $depth--;
+ } elsif (ref($perl) and ref($perl) =~ /HASH/) {
+ $output .= " "x$depth unless ($nospace);
+ $output .= "{\n";
+ my $c = 0;
+ $depth++;
+ for my $key (sort keys %$perl) {
+ $output .= ",\n" if ($c);
+ $output .= " "x$depth;
+ $output .= perl2prettyJSON($key)." : ".perl2prettyJSON($$perl{$key}, undef, 1);
+ $c++;
+ }
+ $depth--;
+ $output .= "\n";
+ $output .= " "x$depth;
+ $output .= '}';
+ } elsif (ref($perl) and ref($perl) =~ /ARRAY/) {
+ $output .= " "x$depth unless ($nospace);
+ $output .= "[\n";
+ my $c = 0;
+ $depth++;
+ for my $part (@$perl) {
+ $output .= ",\n" if ($c);
+ $output .= " "x$depth;
+ $output .= perl2prettyJSON($part);
+ $c++;
+ }
+ $depth--;
+ $output .= "\n";
+ $output .= " "x$depth;
+ $output .= "]";
+ } elsif (ref($perl) and ref($perl) =~ /CODE/) {
+ $output .= perl2prettyJSON(undef,$perl->(), $nospace);
+ } elsif (ref($perl) and "$perl" =~ /^([^=]+)=(\w{4,5})\(0x/) {
+ my $type = $2;
+ my $name = $1;
+ register_class_hint(undef, name => $name, hint => $name, type => lc($type));
+ $output .= perl2prettyJSON(undef,$perl);
+ } else {
+ $perl = NFC($perl);
+ $perl =~ s/\\/\\\\/sgo;
+ $perl =~ s/"/\\"/sgo;
+ $perl =~ s/\t/\\t/sgo;
+ $perl =~ s/\f/\\f/sgo;
+ $perl =~ s/\r/\\r/sgo;
+ $perl =~ s/\n/\\n/sgo;
+ $perl =~ s/([\x{0080}-\x{fffd}])/sprintf('\u%0.4x',ord($1))/sgoe;
+ $output .= " "x$depth unless($nospace);
+ if (length($perl) < 10 and $perl =~ /^(?:\+|-)?\d*\.?\d+$/o and $perl !~ /^(?:\+|-)?0\d+/o ) {
+ $output = $perl;
+ } else {
+ $output = '"'.$perl.'"';
+ }
+ }
+
+ return $output;
+}
+
+1;
Added: PINES-Examples/trunk/scripts/logtools/README
===================================================================
--- PINES-Examples/trunk/scripts/logtools/README (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/README 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,46 @@
+Copyright (C) 2008-2010 Equinox Software, Inc
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+===============================================================================
+
+This kit consists of several files that come in quite handy during Evergreen
+log analysis. Note that these scripts were developed for a specific Evergreen
+installation - and thus a specific log format configuration. While many
+logs are consistent across multiple Evergreen installs, your mileage may vary.
+
+These scripts were designed to be installed in /root/stats - if installed
+elsewhere, you will need to modify the scripts to point to the new path.
+
+Generally speaking, 'cd' to your log directory and run the script. EX:
+#$ cd /var/logs/evergreen/prod/2010/03/16
+#$ /root/stats/searches.sh
+
+Other scripts are run with the log file as a command variable. Ex:
+#$ /root/stats/pglong.sh /var/log/evergreen/prod/2010/03/16/pg.18.log
+
+Some scripts have hard-coded options in them that will need to be changed for
+your environment. For example, in 'ordered_queries.sh', there are a few 'grep -v'
+commands to exclude the reporting system and the SLON replication.
+
+
+This README is not meant in any way to be exhaustive - it was, in reality,
+thrown together quite quickly so we can get these scripts out to the community.
+It is expected that many Evergreen SysAdmins will understand what they are doing
+and understand the changes that will be needed to customize them to their
+local environment. For any assistance, the Evergreen Community is often
+willing to lend a hand. Visit http://www.evergreen-ils.org/ for more information.
+
+
Added: PINES-Examples/trunk/scripts/logtools/autovac-spikes.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/autovac-spikes.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/autovac-spikes.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/sh
+zgrep autovacuum $@ |grep fw4|grep sparkle -A1|cut -f7 -d' '|grep -v '\-\-'|perl -e '$t=0;$x=1;$p;while(<>){chomp;$c=$_;if($c=~/^..:(..):..$/o){$t=$1}else{next};if($x%2){$p=$t}else{print($t-$p);print("\t$c\n")}$x++}'|perl -e 'while(<>){if(/^(\d+)/o){print($_)if($1>1)}}'
Property changes on: PINES-Examples/trunk/scripts/logtools/autovac-spikes.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/checkin_count.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/checkin_count.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/checkin_count.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/bash
+grep osrf_json activity.log | grep -c "open-ils.circ.checkin"
Property changes on: PINES-Examples/trunk/scripts/logtools/checkin_count.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/checkout_count.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/checkout_count.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/checkout_count.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/bash
+grep osrf_json activity.log | egrep "open-ils.circ.renew|open-ils.circ.checkout" | grep -cv "checkout.permit"
Property changes on: PINES-Examples/trunk/scripts/logtools/checkout_count.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/ip_count.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/ip_count.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/ip_count.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/bash
+grep osrf_json activity.log | cut -d' ' -f 6 | sort | uniq -c | sort -n
Property changes on: PINES-Examples/trunk/scripts/logtools/ip_count.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/method_by_hour.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/method_by_hour.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/method_by_hour.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+for h in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23; do
+ echo -e "HOUR = $h\n"
+ grep " $h:[0-9][0-9]:[0-9][0-9]" activity.log | grep osrf_json | cut -d' ' -f 9 | sort | uniq -c | sort -n
+ echo ""
+done;
+
Property changes on: PINES-Examples/trunk/scripts/logtools/method_by_hour.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/method_count.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/method_count.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/method_count.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/bash
+grep osrf_json activity.log | cut -d' ' -f 9 | sort | uniq -c | sort -n
Property changes on: PINES-Examples/trunk/scripts/logtools/method_count.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/non_ws_logins.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/non_ws_logins.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/non_ws_logins.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+grep "successful login" activity.log | cut -d' ' -f 8,10 | egrep "workstation=$" | sort | uniq | wc -l
Property changes on: PINES-Examples/trunk/scripts/logtools/non_ws_logins.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/null-requests.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/null-requests.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/null-requests.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,10 @@
+#!/bin/bash
+#export logfile=/var/log/remote/prod/$(date +%Y/%m/%d)/gateway.$(date +%H).log && export tmp=$(tempfile) && grep "Returning NULL" $logfile | cut -d: -f8 | cut -d" " -f1 > $tmp && grep -f $tmp $logfile | grep "method=" | cut -d" " -f9,3 | sort -k6 | uniq -s6 -c && rm $tmp && unset logfile && unset tmp
+
+#PATH="/var/log/remote/prod/$(date +%Y/%m/%d)";
+#for key in $(grep NULL "$PATH/gateway.$(date +%H).log" | cut -d':' -f 8 | cut -d']' -f1); do grep -m 1 $key $PATH/activity.log ; done | cut -d' ' -f 2,3,9
+
+LOGPATH=/var/log/remote/prod/$(date +%Y/%m/%d);
+for key in $(grep NULL $LOGPATH/gateway.$(date +%H).log | cut -d':' -f 8 | cut -d']' -f1); do
+ grep -m 1 $key $LOGPATH/activity.log ;
+done | cut -d' ' -f 2,3,9
Property changes on: PINES-Examples/trunk/scripts/logtools/null-requests.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/ordered_queries.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/ordered_queries.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/ordered_queries.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+grep -Hn duration: $@ | \
+ grep -v 10.1.0.12 | \
+ grep -v vacuum | \
+ grep -v _prod_replica_set | \
+ grep -vi commit | \
+ awk '{print $16 " " $1}' | \
+ sed 's/:/ /g' | \
+ awk '{print $1 "\t" $2 ":" $3}' | \
+ sort -run | \
+ perl -e '
+ while(<>){
+ ($d,$f,$l)=/^([\d\.]+)\s+(\S+):(\d+)$/o;
+
+ my $first=`head -n$l $f|tail -n1|sed "s/\\\\^I/\t/g"|cut -d" " -f3-`;
+ my ($h,$n,$t)=$first=~/^(\S+) postgres\[\d+\]: \[(\d+)-\d+\] \[([^]]+)\]/smo;
+
+ $le=$l+500;
+ @c=`head -n$le $f|tail -n501|grep $n|sed "s/\\\\^I/\t/g"|cut -d" " -f5-`;
+
+
+ s/^\s*$//o for (@c);
+
+ #my @tmp;
+ #for (@c) {
+ # push @tmp, $_ if (/^\[$n-/);
+ #}
+ #@c = @tmp;
+
+ s/^\[\d+\-\d+\]//o for (@c);
+ s/\s*$//o for (@c);
+ s/^$//o for (@c);
+
+ my $line = join " ", map {$_ ? ($_) : ()} @c;
+ $line =~ s/^.+?ment: //go;
+ $line =~ s/\s+/ /go;
+ $line =~ s/^\s+//go;
+ $line =~ s/\s+$//go;
+ print("$h $d : $t : $line\n");
+ }
+ '
Property changes on: PINES-Examples/trunk/scripts/logtools/ordered_queries.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/pglong.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/pglong.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/pglong.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+LOG=$1
+
+tail -f "$LOG" | grep "duration: " | cut -d' ' -f 4,17 | perl -e 'while(<>) { chomp; my($pid,$time) = split(/ /,"$_"); if( $time > 999 ) { print "$pid $time\n" } }'
Property changes on: PINES-Examples/trunk/scripts/logtools/pglong.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/process.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/process.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/process.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+file=$1;
+end=${file:19:2};
+echo "Processing file: $file"
+if [ "$end" = 'gz' ]; then
+ gzip -cd $file;
+else
+ cat $file;
+fi;
+
Property changes on: PINES-Examples/trunk/scripts/logtools/process.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/req_count.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/req_count.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/req_count.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ -f activity.log ]; then
+ grep osrf_json activity.log | cut -d' ' -f 2 | cut -d':' -f 1 | sort | uniq -c
+else
+ gzip -cd activity.log.gz | grep osrf_json | cut -d' ' -f 2 | cut -d':' -f 1 | sort | uniq -c
+fi;
Property changes on: PINES-Examples/trunk/scripts/logtools/req_count.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/search_counts.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/search_counts.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/search_counts.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo Top Ten search counts by hour
+zgrep multiclass activity.log*|cut -f1-2 -d' '|cut -b1-13|sort|uniq -c|sort -rn|head
+
+echo Top Ten search counts by minute
+zgrep multiclass activity.log*|cut -f1-2 -d' '|cut -b1-16|sort|uniq -c|sort -rn|head
+
+echo Top Ten search counts by second
+zgrep multiclass activity.log*|cut -f1-2 -d' '|cut -b1-19|sort|uniq -c|sort -rn|head
Property changes on: PINES-Examples/trunk/scripts/logtools/search_counts.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/search_session.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/search_session.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/search_session.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,61 @@
+#!/bin/bash
+SESSION=$1
+DEBUG=$2
+[ -z "$SESSION" ] && echo "I need an auth token" && exit;
+
+BASE='/var/log/remote/prod/'
+
+
+function cat_file {
+ file=$1;
+ [ ! -f "$file" ] && file="$file.gz";
+ end=${file:19:2};
+ if [ "$end" = 'gz' ]; then
+ gzip -cd $file;
+ else
+ cat $file;
+ fi;
+}
+
+function set_date {
+ BACK=$1
+ DATE=$(date +%Y-%m-%d --date="$BACK day ago")
+ YEAR=$(date +%Y --date="$BACK day ago")
+ MONTH=$(date +%m --date="$BACK day ago")
+ DAY=$(date +%d --date="$BACK day ago")
+}
+
+
+# --------------------------------------------------------------------
+# Work backwards to find the original login
+# --------------------------------------------------------------------
+
+back=0
+while(true); do
+ set_date $back
+ file="$BASE/$YEAR/$MONTH/$DAY/activity.log"
+ echo "Searching for start of session in $file"
+ LOGIN=$(cat_file $file | grep $SESSION | grep "successful login: " | cut -d' ' -f 1,2,8,10-);
+ [ -n "$LOGIN" ] && break;
+ back=$(expr $back + 1);
+done;
+
+echo -e "\n$LOGIN\n"
+
+
+
+# --------------------------------------------------------------------
+# Now, work forward and grab all session activity
+# --------------------------------------------------------------------
+
+while(true); do
+ set_date $back
+ file="$BASE/$YEAR/$MONTH/$DAY/activity.log"
+ ACTIVITY=$(cat_file $file | grep osrf_json | grep $SESSION | cut -d' ' -f 1,2,9-);
+ echo "$ACTIVITY"
+ back=$(expr $back - 1);
+ [ $back -le 0 ] && break;
+done;
+
+
+
Property changes on: PINES-Examples/trunk/scripts/logtools/search_session.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/searches.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/searches.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/searches.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,22 @@
+#!/bin/bash
+cat activity.log | \
+ grep osrf_json | \
+ grep multiclass | \
+ cut -d' ' -f 10- | \
+ PERL5LIB=/root/stats/ perl -MJSON -MUnicode::Normalize -e '
+ while(<>) {
+ chomp;
+ $line = "$_";
+ $line =~ s/,\s*\d$//og;
+ $line = JSON->JSON2perl($line);
+ $line = $line->{searches};
+ for my $k( keys %{$line} ) {
+ my $term = lc(NFD($line->{$k}->{term}));
+ $term =~ s/^\s*//o;
+ $term =~ s/\s*$//o;
+ $term =~ s/\s+/ /o;
+ $term =~ s/[\x{0080}-\x{ffdf}]+//o;
+ print "$k = $term\n" if ($line->{offset} == 0);
+ }
+ }
+ '
Property changes on: PINES-Examples/trunk/scripts/logtools/searches.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/tail-searches.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/tail-searches.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/tail-searches.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,20 @@
+#!/bin/bash
+tail -f activity.log | \
+ grep osrf_json | \
+ grep multiclass | \
+ cut -d' ' -f 10- | \
+ PERL5LIB=/root/stats/ perl -MJSON -e '
+ while(<>) {
+ chomp;
+ $line = "$_";
+ $line =~ s/,\s*\d$//og;
+ $line = JSON->JSON2perl($line);
+ $line = $line->{searches};
+ print "--\n";
+ for my $k( keys %{$line} ) {
+ $kk = $k;
+ #print "$kk\n";
+ print "$kk = ". $line->{$k}->{term} . "\n";
+ }
+ }
+ '
Property changes on: PINES-Examples/trunk/scripts/logtools/tail-searches.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/top_queries.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/top_queries.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/top_queries.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,63 @@
+#!/bin/sh
+
+_top=$1
+_bottom=$2
+_file=$3
+
+if [ -z "$_top" ]; then
+ _top=10;
+fi
+
+if [ -z "$_bottom" ]; then
+ _bottom=$_top;
+fi
+
+if [ -z "$_file" ]; then
+ _file=pg.*.log;
+fi
+
+export _top
+export _bottom
+
+grep -Hn duration: $_file | \
+ grep -v 10.1.0.12 | \
+ grep -v vacuum | \
+ grep -v _prod_replica_set | \
+ grep -vi commit | \
+ awk '{print $16 " " $1}' | \
+ sed 's/:/ /g' | \
+ awk '{print $1 "\t" $2 ":" $3}' | \
+ sort -run | \
+ head -n $_top | \
+ tail -n $_bottom | \
+ perl -e '
+ while(<>){
+ ($d,$f,$l)=/^([\d\.]+)\s+(\S+):(\d+)$/o;
+ $le=$l+50;
+
+ @c=`head -n$le $f|tail -n51|sed "s/\\\\^I/\t/g"|cut -d" " -f5-`;
+ my ($n,$h)=$c[0]=~/^\[(\d+)-\d+\]\s+(.+?)LOG:.+?ment:(.+)/smo;
+
+
+ (my $first = $c[0]) =~ s/^.+ment:(.+)$/$1/o;
+ $first=~s/\s*$//o;
+ chomp($first);
+
+ s/^\s*$//o for (@c);
+
+ my @tmp;
+ for (@c[1 .. @c - 2]) {
+ push @tmp, $_ if (/^\[$n-/);
+ #warn "$n -- $_" if (/^\[$n-/);
+ }
+ @c = @tmp;
+
+ s/^\[\d+\-\d+\]//o for (@c);
+ s/\s*$//o for (@c);
+ s/^$//o for (@c);
+
+ print(int($. + $ENV{_top} - $ENV{_bottom})." == $d ms : $h". "-"x100 ."\n");
+ print(join "\n", map {$_ ? ($_) : ()} $first, at c);
+ print("\n\n");
+ }
+ '
Property changes on: PINES-Examples/trunk/scripts/logtools/top_queries.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/top_searches.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/top_searches.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/top_searches.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+HOUR=$1
+
+if [ -z $HOUR ]; then
+ HOUR='*';
+fi
+
+if [ "_$SEARCH_DIR_BASE" == "_" ]; then
+ SEARCH_DIR_BASE='.';
+fi
+
+if [ "_$SEARCH_DIR_BASE" == "_" ]; then
+ SEARCH_DIR_BASE='.';
+fi
+
+zgrep 'compiled search is' $SEARCH_DIR_BASE/osrfsys.$HOUR.log*| PERL5LIB=/root/ perl -MJSON -pe \
+'
+ s/^.+compiled search is (.+$)/$1/gso;
+ $x = undef;
+ $x = JSON->JSON2perl($_)->{searches};
+
+ $t = "";
+ for my $k (keys %$x) {
+ $pt = $$x{$k}{term};
+
+ $pt =~ s/[^\w ]//gso;
+ $pt =~ s/\s+/ /gso;
+ $pt =~ s/^\s+//gso;
+ $pt =~ s/\s+$//gso;
+ $pt = lc($pt);
+
+ $k = uc($k);
+ $t .= "$pt\n$pt ($k)\n";
+ }
+
+ $_ = $t;
+' 2>/dev/null | sort -s | uniq -c | sort -snr
+#' | sort | uniq -c | sort -nr | head -n $COUNT;
+
+exit
+
+'
+ s/^.+compiled search is (.+$)/$1/gso;
+ warn $_;
+ $_ = eval { (values(%{JSON->JSON2perl($_)->{searches}}))[0]->{term} };
+ if (!$@) {
+ s/[^\w ]//gso;
+ s/\s+/ /gso;
+ s/^\s+//gso;
+ s/\s+$//gso;
+ $_=lc($_)."\n";
+
+ }
+' 2>/dev/null | sort | uniq -c | sort -nr | head -n $COUNT;
+
Property changes on: PINES-Examples/trunk/scripts/logtools/top_searches.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/utils.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/utils.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/utils.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,19 @@
+#!/usr/bin/bash
+
+
+function cat_files {
+ for arg in $*; do
+ echo "-> $arg" >&2;
+ if [ -f $arg ]; then
+ if [ $(echo "$arg" | grep ".gz") ]; then
+ gzip -cd "$arg"
+ else
+ cat $arg;
+ fi;
+ else
+ gzip -cd "$arg.gz";
+ fi;
+ done;
+}
+
+
Property changes on: PINES-Examples/trunk/scripts/logtools/utils.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/waits.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/waits.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/waits.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,2 @@
+#!/bin/bash
+grep "No children available, waiting..." osrfwarn.log | grep cstore | cut -d' ' -f2 | cut -d':' -f1 | uniq -c
Property changes on: PINES-Examples/trunk/scripts/logtools/waits.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: PINES-Examples/trunk/scripts/logtools/watch_searches.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/watch_searches.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/watch_searches.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,10 @@
+#!/bin/bash
+tail -f activity.log | grep osrf_json | cut -d' ' -f 1,2,3,6,7,9- | grep multiclass | PERL5LIB=/root/stats/ perl -MJSON -e '
+ while($line = <>) {
+ chomp $line;
+ $line =~ s/^.+?multiclass //og;
+ $line = JSON->JSON2perl("[$line]");
+ $line = $line->[0]->{searches};
+ print join(" :: " => map { "$_ =~ $$line{$_}{term}" } sort keys %$line)."\n";
+ }
+ '
Added: PINES-Examples/trunk/scripts/logtools/ws_logins.sh
===================================================================
--- PINES-Examples/trunk/scripts/logtools/ws_logins.sh (rev 0)
+++ PINES-Examples/trunk/scripts/logtools/ws_logins.sh 2010-03-16 20:44:05 UTC (rev 834)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+grep "successful login" activity.log | cut -d' ' -f 8,10 | egrep -v "workstation=$" | sort | uniq | wc -l
Property changes on: PINES-Examples/trunk/scripts/logtools/ws_logins.sh
___________________________________________________________________
Name: svn:executable
+ *
More information about the open-ils-commits
mailing list