#!/usr/bin/perl -w # # # This script will parse the Urchin 6 scheduler history file for errors # for a particular date (default today) and print a notification if # any profile exits with a non-zero exit status. If desired, the runtime # output from each failed task can be printed inline. # # Urchin tasks are recorded with the following exit status and codes: # exit_status=0, exit_code=0: no warnings or errors encountered # exit_status=1, exit_code=0: warning condition encountered # exit_status=2, exit_code=X: error condition encountered, Urchin error # code 'X' stored in exit_code # exit_status=3, exit_code=X: error condition encountered; scheduler unable # to execute log processing engine properly, # error condition encountered, Urchin error # code 'X' stored in exit_code # # U6: # uipt_status=1 - OK # uipt_status=2 - WARN # uipt_status=3 - ERR # # Usage: u6scan_historylog.pl [--date CCYYMMDD] [--urchinpath /path/to/urchin] # [--nowarnings] [--nodata-warn] [--showrunlog] [--summary] # [--warn] | [--help] # # Where: # --help shows this message # --urchinpath specifies the path of the Urchin 6 distribution # Default: /usr/local/urchin # --date specifies the date of tasks to scan for, specified in # CCYYMMDD format or CCYYMMDDHH if --hourly selected. # Default: today # --hourly specifies an hourly date range instead of an entire day. # Default: off # --nowarnings disables printing of messages for tasks that just # have warnings and not errors # Default: off # --nodata-warn enables printing of profiles that ran, but did not process # any data. # Default: off # --showrunlog causes the runtime log for profiles with errors to be # printed along with the error notification # Default: off # --summary causes the total number of profiles processed for the # requested date to be printed # --warn causes a warning to be printed if no history entries are # found with the date requested. # Default: off # --account account name. Default: %28NONE%29 # # Exit status codes: # 0: Profiles processed for the given date, but no errors detected # 1: No profiles processed for the given date # 2: Invalid command line or other unrecoverable error encountered # 3: Errors and/or warnings found for profiles processed on the given date # # Copyright (c) 2003, Urchin Software Corporation. # RCS ID: $Revision: 1.5 $ # 2008, Rewritten, $Revision: 1.2 $ # use strict; use Getopt::Long; use DBI(); use Time::localtime; use Time::Local; # Initialize variables my $help = 0; my $gstatus = 0; my $selecteddate = ""; my $hourly = 0; my $nowarnings = 0; my $showrunlog = 0; my $summary = 0; my $nodatawarn = 0; my $urchinpath = "/usr/local/urchin"; my $account = "%28NONE%29"; my @logentries = (); my $datadir = ""; my $do_warn = 0; my $historyfile = ""; my $warnings = 0; my $errors = 0; my $empty = 0; my %config; # Set output to flush after each print line $| = 1; # Read in the options $gstatus = GetOptions( 'urchinpath=s' => \$urchinpath, 'date=s' => \$selecteddate, 'nowarnings' => \$nowarnings, 'account' => \$account, 'showrunlog' => \$showrunlog, 'summary' => \$summary, 'warn' => \$do_warn, 'nodata-warn' => \$nodatawarn, 'help' => \$help, 'hourly' => \$hourly ); if ($help) { &Usage; exit(0); } if ( $gstatus ne 1 ) { &Usage; exit(2); } $datadir = $urchinpath . "/data"; # Checking the urchin directory if ( folder_check($urchinpath) ) { print "Attempted to locate the urchin root directory, but failed.\n"; exit(2); } my $urchinconfig = $urchinpath . "/etc/urchin.conf"; # locate of urchin.conf &parseUrchinConf( $urchinconfig, \%config ); # Calculate the date. if ( ( !defined($selecteddate) ) || $selecteddate eq "" ) { if ($hourly) { $selecteddate = convert_epoch_hourly( time() ); } else { $selecteddate = convert_epoch( time() ); } } else { $selecteddate = reformatTime($selecteddate); } if ( check_date($selecteddate) ) { print "The date you entered was invalid: \"$selecteddate\".\n"; &Usage; exit(2); } do_u6history( \%config ); check_tasks_issues(); print "\n"; # Print out data based on command line data. if ($summary) { print_u6title( "Urchin Runtime Report for $selecteddate", 85 ); } if ( scalar( @logentries == 0 ) ) { if ($do_warn) { print "+ WARNING: No profiles run on $selecteddate\n"; } exit(1); } else { if ( !$nowarnings && $warnings ) { print_u6type("warnings"); } if ( $errors ) { print_u6type("errors"); } if ( $nodatawarn && $empty ) { print_u6empties(); } if ( $summary ) { print_u6summary(); } } if ( ( $warnings || $errors ) > 0 ) { exit(3); } else { exit(0); } #------------------------------------------------------------------------------ sub print_u6runlog { my $entrynumber = $_[0]; my $profilename = ""; my $runlog = ""; $profilename = $logentries[$entrynumber]->{"ucpr_name"}; print_u6title( $profilename, 85, '+' ); $runlog = "$datadir/history/$account/$profilename/" . mk_logname( $logentries[$entrynumber]->{"utpt_process_start_time"} ) . ".log"; if ( file_check($runlog) ) { return (1); } if ( !open( RUNLOG, $runlog ) ) { print "Failed to open: $!\n"; return (1); } else { while () { print; } close(RUNLOG); print_bar( "+", 85 ); print "\n"; } return (0); } sub print_u6empties { my $counter = 0; my ( $profile_name, $sys_pid, $run_time ); ( $~, $^, $-, $% ) = ( 'NODATA_FORMAT', 'NODATA_TOP', 0, 0 ); print_header( "Profiles with no data", 85 ); for ( $counter = 0 ; $counter < @logentries ; $counter++ ) { if ( $logentries[$counter]->{"uipt_exit_code"} eq "0" && $logentries[$counter]->{"uspt_data_hits"} eq "0" ) { $profile_name = $logentries[$counter]->{"ucpr_name"}; $sys_pid = $logentries[$counter]->{"my_lockid"}; $run_time = $logentries[$counter]->{"utpt_process_start_time"}; write(); } } &print_bar( "-", 85 ); print "\n\n"; format NODATA_TOP = Profile Name System & Pid Run Time ===================================================================================== . format NODATA_FORMAT = @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<@>>>>>>>>>>>>>>>>>>>>>>>>>>>@>>>>>>>>>>>>>>>>>>>> $profile_name, $sys_pid, $run_time . } sub print_u6type { my $type = $_[0]; my $counter = 0; my $status = 1; my @entries = (); my ( $profile_name, $run_time ); if ( $type eq "warnings" ) { $status = "2"; print_header( "Profiles with warnings", 85 ); } elsif ( $type eq "errors" ) { $status = "3"; print_header( "Profiles with errors", 85 ); } ( $~, $^, $-, $% ) = ( 'TYPES_FORMAT', 'TYPES_TOP', 0, 0 ); for ( $counter = 0 ; $counter < @logentries ; $counter++ ) { if ( $logentries[$counter]->{"uipt_status"} eq "$status" ) { $profile_name = $logentries[$counter]->{"ucpr_name"}; $run_time = $logentries[$counter]->{"utpt_process_start_time"}; push( @entries, $counter ); write(); } } &print_bar( '-', 85 ); print "\n"; if ($showrunlog) { foreach (@entries) { print_u6runlog($_); } } format TYPES_TOP = Profile Name Run Time ===================================================================================== . format TYPES_FORMAT = @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<@>>>>>>>>>>>>>>>>>>>> $profile_name, $run_time . } sub do_u6history { my $config = $_[0]; my %hash; my ( $btime, $etime ); my $dbi = $config{'SQLType'}; if ( $dbi eq "postgresql" ) { $dbi = "Pg"; } my $dsn = sprintf( "DBI:%s:database=%s;host=%s", $dbi, $config{'SQLDatabase'}, $config{'SQLServer'} ); my $dbh = DBI->connect( $dsn, $config{'SQLUsername'}, $config{'SQLPassword'}, { 'RaiseError' => 1 } ); my $sth = $dbh->prepare( sprintf( "SELECT * FROM `uprofiles_tasks` WHERE `utpt_mtime` LIKE '%s%%' ORDER BY `utpt_process_start_time`", $selecteddate ) ); if ( !$sth ) { die "Error:" . $dbh->errstr . "\n"; } if ( !$sth->execute ) { die "Error:" . $sth->errstr . "\n"; } my @names = @{ $sth->{NAME_lc} }; my $machine_name; while ( my $row = $sth->fetchrow_arrayref ) { @hash{@names} = @$row; push( @logentries, &new_u6entry(%hash) ); } $dbh->disconnect; return (0); } sub new_u6entry { my %entry = @_; my ( $btime, $etime ); if ( $entry{'uipt_process_percent_completed'} ) { $btime = &to_timestamp( $entry{'utpt_process_start_time'} ); $etime = &to_timestamp( $entry{'utpt_process_end_time'} ); } else { $btime = 0; $etime = 0; } $entry{'my_btime'} = $btime; $entry{'my_etime'} = $etime; $entry{'my_ptime'} = $etime - $btime; if ( !defined( $entry{'ucpt_machine_name'} ) ) { $entry{'ucpt_machine_name'} = "-"; $entry{"my_lockid"} = "(command line)"; } else { $entry{'my_lockid'} = $entry{'ucpt_machine_name'}; $entry{'my_lockid'} = &extractName( $entry{'ucpt_machine_name'} ) . "." . $entry{'uipt_process_id'}; } if ( !defined( $entry{"uipt_exit_code"} ) ) { $entry{"uipt_exit_code"} = 0; } if ( !defined( $entry{"uipt_process_id"} ) ) { $entry{"uipt_process_id"} = 0; } return ( \%entry ); } sub print_u6summary { my %totals; my $counter = 0; my $counter2 = 0; my ( $sysname, $log_hits, $hits, $log_lines, $lines, $bytes, $time, $speed, $valid, $_valid, $_warnings, $_errors, $_empty ); for ( $counter = 0 ; $counter < @logentries ; $counter++ ) { if ( $logentries[$counter]->{"uipt_exit_code"} eq "0" ) { $totals{ $logentries[$counter]->{"my_lockid"} }[0] += $logentries[$counter]->{"uspt_data_hits"}; $totals{ $logentries[$counter]->{"my_lockid"} }[1] += $logentries[$counter]->{"uspt_data_lines"}; $totals{ $logentries[$counter]->{"my_lockid"} }[2] += $logentries[$counter]->{"uspt_data_bytes"}; $totals{ $logentries[$counter]->{"my_lockid"} }[3] += $logentries[$counter]->{"my_ptime"}; } } print_header( "Log processing statistics by system", 85 ); ( $~, $^, $-, $% ) = ( 'SUMMARY_FORMAT_1', 'SUMMARY_TOP_1', 0, 0 ); foreach ( sort keys(%totals) ) { $sysname = $_; $log_hits = $totals{$_}[0]; $hits = reduce_normal( $totals{$_}[0] ); $log_lines = $totals{$_}[1]; $lines = reduce_normal( $totals{$_}[1] ); $bytes = reduce_bytes( $totals{$_}[2] ); $time = reduce_time( $totals{$_}[3] ); $speed = reduce_speed( calc_proc_rate( $totals{$_}[2], $totals{$_}[3] ) ); write(); $counter2++; } if ( $counter2 > 1 ) { print_bar( "-", 85 ); $sysname = "Total"; $log_hits = get_total( "0", %totals ); $hits = reduce_normal( get_total( "0", %totals ) ); $log_lines = get_total( "1", %totals ); $lines = reduce_normal( get_total( "1", %totals ) ); $bytes = reduce_bytes( get_total( "2", %totals ) ); $time = reduce_time( get_total( "3", %totals ) ); $speed = reduce_speed( calc_proc_rate( get_total( "2", %totals ), get_total( "3", %totals ) ) ); write(); } print "\n\n"; %totals = (); $counter2 = 0; for ( $counter = 0 ; $counter < @logentries ; $counter++ ) { if ( $logentries[$counter]->{"uipt_status"} eq "3" ) { $totals{ $logentries[$counter]->{"my_lockid"} }[2] += 1; } elsif ( $logentries[$counter]->{"uipt_status"} eq "2" ) { $totals{ $logentries[$counter]->{"my_lockid"} }[1] += 1; } elsif ( $logentries[$counter]->{"uipt_status"} eq "1" ) { if ( $logentries[$counter]->{"uspt_data_hits"} eq "0" ) { $totals{ $logentries[$counter]->{"my_lockid"} }[3] += 1; } else { $totals{ $logentries[$counter]->{"my_lockid"} }[0] += 1; } } } print_header( "Profile processing statistics by Urchin Log Processing System", 85 ); ( $~, $^, $-, $% ) = ( 'SUMMARY_FORMAT_2', 'SUMMARY_TOP_2', 0, 0 ); my ( $key, $val, @_val ); while ( ( $key, $val ) = each(%totals) ) { $sysname = $key; @_val = @{$val}; $_valid = $_val[0] ? $_val[0] : 0; $_warnings = $_val[1] ? $_val[1] : 0; $_errors = $_val[2] ? $_val[2] : 0; $_empty = $_val[3] ? $_val[3] : 0; write(); $counter2++; } if ( $counter2 > 1 ) { &print_bar( '-', 85 ); $sysname = "Total Profiles: " . scalar(@logentries); $_valid = scalar(@logentries) - ( $warnings + $errors + $empty ); $_errors = $errors; $_warnings = $warnings; $_empty = $empty; write(); } print "\n\n"; format SUMMARY_TOP_1 = System & Pid Log Hits Hits Log Lines Lines Bytes Time Speed ===================================================================================== . format SUMMARY_FORMAT_1 = @<<<<<<<<<<<<<<<<<<<@>>>>>>>@>>>>>>>@>>>>>>>>>@>>>>>>>>@>>>>>>>>>@>>>>>>>>@>>>>>>>>>> $sysname, $log_hits, $hits,$log_lines, $lines, $bytes, $time, $speed . format SUMMARY_TOP_2 = System & Pid Valid Warnings Errors No Data ===================================================================================== . format SUMMARY_FORMAT_2 = @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<@>>>>>>>>>>@>>>>>>>>>>@>>>>>>>>>>@>>>>>>>>> $sysname, $_valid, $_warnings, $_errors, $_empty . } sub print_bar { my $bar = $_[0] ? $_[0] : "-"; my $width = $_[1] ? $_[1] : 80; print $bar x $width, "\n"; } sub print_bar_fx { my $width = $_[0] ? $_[0] : 80; my $i; for ( $i = 0 ; $i < $width ; $i++ ) { $i % 2 ? print '-' : print '='; } print "\n"; } sub print_u6title { my $lineinfo = $_[0]; my $whdth = $_[1] ? $_[1] : 80; my $bar = $_[2] ? $_[2] : "="; $~ = 'TITLE_FORMAT'; print $bar x $whdth, "\n"; write(); print $bar x $whdth, "\n"; format TITLE_FORMAT = @|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| $lineinfo . } sub check_tasks_issues { my $counter = 0; for ( $counter = 0 ; $counter < @logentries ; $counter++ ) { if ( $logentries[$counter]->{"uipt_status"} eq "3" ) { $errors++; } elsif ( $logentries[$counter]->{"uipt_status"} eq "2" ) { $warnings++; } elsif ( $logentries[$counter]->{"uspt_data_hits"} eq "0" && $logentries[$counter]->{"uipt_status"} eq "1" ) { $empty++; } } } sub file_check { my $filename = $_[0]; my $status = 0; if ( !-e "$filename" ) { print "File: \"$filename\" could not be found.\n"; $status = 1; } elsif ( !-r $filename ) { print "File: \"$filename\" cannot be opened for reading.\n"; $status = 1; } return ($status); } sub folder_check { my $foldername = $_[0]; my $status = 0; if ( !-e $foldername ) { print "Folder: \"$foldername\" could not be found: $!\n"; $status = 1; } elsif ( !-r $foldername ) { print "folder: \"$foldername\" cannot be opened for reading $!\n"; $status = 1; } return ($status); } sub get_datadir { my $path = $_[0]; my $status = 0; if ( !file_check("$path/etc/urchin.conf") ) { $status = 1; } if ( !open( URCHINCONFIG, "$path/etc/urchin.conf" ) ) { print "Could not open the configuration file for reading: $!\n"; $status = 1; } else { while () { if ( substr( $_, 0, 1 ) eq '#' ) { next; } chomp; if ( index( $_, "dataDirectory:" ) eq -1 ) { next; } $datadir = substr( $_, index( $_, ":" ) + 1 ); $datadir =~ s/[ \t]//g; last; } if ( ( !defined($datadir) ) || $datadir eq "" ) { $datadir = "$urchinpath/data"; } unless ( close(URCHINCONFIG) ) { print "Could not close urchin configuration file: $!\n"; } $status = 0; } if ( ( !defined($datadir) ) || $datadir eq "" ) { $status = 1; } return ($status); } sub reformatTime { my $rawtime = $_[0]; if ($hourly) { $rawtime =~ /(\d{4})(\d{2})(\d{2})(\d{2})/; if(!defined($4)){ print "Invalid date format. Should be CCYYMMDDHH if --hourly selected.\n"; exit(2); die ""; } else { return "$1-$2-$3 $4"; } } else { $rawtime =~ /(\d{4})(\d{2})(\d{2})/; return "$1-$2-$3"; } } sub convert_epoch { my $rawtime = $_[0]; my $formatedtime = ""; if ( ( !defined($rawtime) ) || $rawtime eq "" ) { $formatedtime = 0; } else { $rawtime = localtime($rawtime); $formatedtime = sprintf( "%04d-%02d-%02d", $rawtime->year + 1900, $rawtime->mon + 1, $rawtime->mday ); } return ($formatedtime); } sub convert_epoch_hourly { my $rawtime = $_[0]; my $formatedtime = ""; if ( ( !defined($rawtime) ) || $rawtime eq "" ) { $formatedtime = 0; } else { $rawtime = localtime($rawtime); $formatedtime = sprintf( "%04d-%02d-%02d %02d", $rawtime->year + 1900, $rawtime->mon + 1, $rawtime->mday, $rawtime->hour ); } return ($formatedtime); } sub convert_epoch_full { my $rawtime = $_[0]; my $formatedtime = ""; if ( ( !defined($rawtime) ) || $rawtime eq "" ) { $formatedtime = 0; } else { $rawtime = localtime($rawtime); $formatedtime = sprintf( "%04d-%02d-%02d %02d:%02d:%02d", $rawtime->year + 1900, $rawtime->mon + 1, $rawtime->mday, $rawtime->hour, $rawtime->min, $rawtime->sec ); } return ($formatedtime); } sub check_date { my $date = $_[0]; my $status = ""; if ( ( !defined($date) ) || $date eq "" ) { $status = 1; } elsif ($hourly) { if ( !( $date =~ /\d{4}-\d{2}-\d{2}\s\d{2}/ ) ) { $status = 1; } else { $status = 0; } } elsif ( !$date =~ /\d{4}-\d{2}-\d{2}/ ) { $status = 1; } else { $status = 0; } return ($status); } sub calc_proc_rate { my $bytes = $_[0]; my $seconds = $_[1]; return ( $bytes / $seconds ); } sub reduce_speed { my $rate = $_[0]; if ( $rate < 1024 ) { return ("$rate B/s"); } elsif ( $rate < 1048576 ) { $rate = ( $rate / 1024 ); return ( sprintf( "%.2fKB/s", $rate ) ); } elsif ( $rate < 1073741824 ) { $rate = ( $rate / 1048576 ); return ( sprintf( "%.2fMB/s", $rate ) ); } else { $rate = ( $rate / 1073741824 ); return ( sprintf( "%.2fGB/s", $rate ) ); } } sub reduce_time { my $seconds = $_[0]; my $mins = 0; my $hours = 0; if ( $seconds > 3600 ) { $hours = int( $seconds / 3600 ); $seconds -= 3600 * $hours; } if ( $seconds > 60 ) { $mins = int( $seconds / 60 ); $seconds -= 60 * $mins; } return ( sprintf( "%02d:%02d:%02d", $hours, $mins, $seconds ) ); } sub reduce_bytes { my $size = $_[0]; if ( $size < 1024 ) { return ("$size"); } elsif ( $size < 1048576 ) { $size = ( $size / 1024 ); return ( sprintf( "%.2fKB", $size ) ); } elsif ( $size < 1073741824 ) { $size = ( $size / 1048576 ); return ( sprintf( "%.2fMB", $size ) ); } else { $size = ( $size / 1073741824 ); return ( sprintf( "%.2fGB", $size ) ); } } sub reduce_normal { my $size = $_[0]; if ( ( !defined($size) ) || $size == 0 ) { $size = $size; } if ( $size < 1000 ) { return ("$size"); } elsif ( $size < 1000000 ) { $size = ( $size / 1000 ); return ( sprintf( "%.2fK", $size ) ); } elsif ( $size < 10000000000 ) { $size = ( $size / 1000000 ); return ( sprintf( "%.2fM", $size ) ); } else { $size = ( $size / 10000000000 ); return ( sprintf( "%.2fB", $size ) ); } } sub print_header { my $lineinfo = $_[0]; my $whdth = $_[1] ? $_[1] : 80; &print_bar('=',$whdth); print "Summary: $lineinfo\n"; &print_bar_fx($whdth); } sub get_total { my ( $field, %temphash ) = @_; my $total = 0; foreach ( keys(%temphash) ) { $total += $temphash{$_}[$field]; } return ($total); } sub print_runlog { my $entrynumber = $_[0]; print $entrynumber, "\n"; return (0); my $profilename = ""; my $runlog = ""; $profilename = $logentries[$entrynumber]->{"ct_name"}; print "-" x 80, "\n"; print "-" x ( 40 - ( ( length($profilename) / 2 ) + 1 ) ), " ", $profilename, " ", "-" x ( 40 - ( ( length($profilename) / 2 ) + 1 ) ), "\n"; print "-" x 80, "\n"; $runlog = "$datadir/history/$account/$profilename/" . $logentries[$entrynumber]->{"cd_btime"} . ".log"; if ( file_check($runlog) ) { return (1); } if ( !open( RUNLOG, $runlog ) ) { print "Failed to open: $!\n"; return (1); } else { while () { print; } close(RUNLOG); print "-" x 80, "\n\n"; } return (0); } sub to_timestamp { my $str = $_[0]; $str =~ m!(\d{4})-(\d{2})-(\d{2})\s(\d{2}):(\d{2}):(\d{2})!; my $timestamp = timelocal( $6, $5, $4, $3, $2, $1 ); return ($timestamp); } sub mk_logname { my $ts = $_[0]; $ts =~ s/-|://g; $ts =~ s/\s/_/g; return ($ts); } sub extractName { $_[0] =~ /^([A-z0-9_\-9999]+)\./; return ($1); } #-------------------------------------------------------------------------------- # read and parse urchin.config; return cofig entries as hash #-------------------------------------------------------------------------------- sub parseUrchinConf { my $file_name = $_[0]; if ( !defined $file_name || !-r $file_name ) { die "Error: config file \"" . $file_name . "\" doesn't exist\n"; } &parse_config_file( $file_name, $_[1] ); } #------------------------------------------------------------------------------- # parse_config_file(file, &global) # Parses config file into a list of hashes, # each representing one urchin config entry #------------------------------------------------------------------------------- sub parse_config_file { my $line; # Tokenize the file open( FILE, $_[0] ) or die "Can't open $_[0]: $!"; while () { s/#.*//; next if /^(\s)*$/; chomp; $line = $_; if ( $line =~ /^\s*([^"]*):\s+(.*)$/ ) { $_[1]{$1} = "$2"; } } close(FILE); } sub Usage { print "Usage: $0 [--date CCYYMMDD] [--urchinpath /path/to/urchin] [nodata-warn] [--nowarnings] [--showrunlog] [--summary] [--warn] | [--help] Where: --help shows this message --urchinpath specifies the path of the Urchin 6 distribution Default: /usr/local/urchin --date specifies the date of tasks to scan for, specified in CCYYMMDD format or CCYYMMDDHH if --hourly selected. Default: today --hourly specifies an hourly date range instead of an entire day. Default: off --nodata-warn enables printing of profiles that ran, but did not process any data. Default: off --nowarnings disables printing of messages for tasks that just have warnings and not errors Default: off --showrunlog causes the runtime log for profiles with errors to be printed along with the error notification Default: disabled --summary causes the total number of profiles processed for the requested date to be printed --warn causes a warning to be printed if no history entries are found with the date requested. Default: off --account account name. Default: %28NONE%29 \n\n"; }