From 3cd7353a9f96adba1f51ed85d4dd6096aa26ffbe Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sun, 13 Aug 2017 13:07:37 +0100 Subject: [PATCH] [Feature] Add json output for rspamd_stats --- utils/rspamd_stats.pl | 229 +++++++++++++++++++++++++++++++----------- 1 file changed, 172 insertions(+), 57 deletions(-) diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl index 95d04b414..46ad3a652 100755 --- a/utils/rspamd_stats.pl +++ b/utils/rspamd_stats.pl @@ -22,6 +22,7 @@ my $endTime; my $num_logs; my $exclude_logs = 0; my $man = 0; +my $json = 0; my $help = 0; # Associate file extensions with decompressors @@ -46,6 +47,7 @@ GetOptions( "end=s" => \$endTime, "num-logs|n=i" => \$num_logs, "exclude-logs|x=i" => \$exclude_logs, + "json|j" => \$json, "help|?" => \$help, "man" => \$man ) or pod2usage(2); @@ -119,94 +121,206 @@ else { my $total_ham = $total - ($total_spam + $total_junk); -if ($total > 0) { - while (my ($s, $r) = each(%sym_res)) { - if ($r->{hits} > 0) { - my $th = $r->{hits}; - my $sh = $r->{spam_hits}; - my $jh = $r->{junk_hits}; - my $hh = $r->{hits} - $sh - $jh; - my $htp = $hh * 100.0 / $total_ham if $total_ham != 0; - my $stp = $sh * 100.0 / $total_spam if $total_spam != 0; - my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0; - - printf "%s avg. weight %.3f, hits %d(%.3f%%): +if ($json) { + print "{"; + &Summary(); + print '"symbols":{'; + &SymbolsStat(); + print "}}\n"; +} +else { + &SymbolsStat(); + &Summary(); +} + +exit; + +sub SymbolsStat() { + if ($total > 0) { + my $has_comma = 0; + while (my ($s, $r) = each(%sym_res)) { + if ($r->{hits} > 0) { + my $th = $r->{hits}; + my $sh = $r->{spam_hits}; + my $jh = $r->{junk_hits}; + my $hh = $r->{hits} - $sh - $jh; + my $htp = $hh * 100.0 / $total_ham if $total_ham != 0; + my $stp = $sh * 100.0 / $total_spam if $total_spam != 0; + my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0; + + if ($json) { + if ($has_comma) { + print ","; + } + else { + $has_comma = 1; + } + print "\"$s\":{"; + JsonObjectElt("avg_weight", $r->{'weight'},"%.4f"); + print ","; + JsonObjectElt("hits", $th, "%d"); + print ","; + JsonObjectElt("hits_percentage", $th/$total, "%.4f"); + print ","; + JsonObjectElt("spam_hits", $sh, "%d"); + print ","; + JsonObjectElt("spam_to_total", $sh/$th, "%.4f"); + print ","; + JsonObjectElt("spam_percentage", $stp/100.0 || 0, "%.4f"); + print ","; + JsonObjectElt("ham_hits", $hh, "%d"); + print ","; + JsonObjectElt("ham_to_total", $hh/$th, "%.4f"); + print ","; + JsonObjectElt("ham_percentage", $htp/100.0 || 0, "%.4f"); + print ","; + JsonObjectElt("junk_hits", $jh, "%d"); + print ","; + JsonObjectElt("junk_to_total", $jh/$th, "%.4f"); + print ","; + JsonObjectElt("junk_percentage", $jtp/100.0 || 0, "%.4f"); + } + else { + printf "%s avg. weight %.3f, hits %d(%.3f%%): Ham %7.3f%%, %6d/%-6d (%7.3f%%) Spam %7.3f%%, %6d/%-6d (%7.3f%%) Junk %7.3f%%, %6d/%-6d (%7.3f%%) ", - $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ), - ( $hh / $th * 100 ), $hh, $total_ham, ( $htp or 0 ), - ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ), - ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 ); + $s, $r->{weight} / $r->{hits}, $th, ($th / $total * 100), + ($hh / $th * 100), $hh, $total_ham, ($htp or 0), + ($sh / $th * 100), $sh, $total_spam, ($stp or 0), + ($jh / $th * 100), $jh, $total_junk, ($jtp or 0); + } - my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam; - my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk; + my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam; + my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk; - if ($r->{weight} != 0) { - if ($r->{weight} > 0) { - printf " + if ($r->{weight} != 0) { + if (!$json) { + if ($r->{weight} > 0) { + printf " Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%) Spam changes / total spam hits: %6d/%-6d (%7.3f%%) Junk changes (ham -> junk): %6d/%-6d (%7.3f%%) Junk changes / total junk hits: %6d/%-6d (%7.3f%%) ", - $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ), - $r->{spam_change}, $total_spam, ( $schp or 0 ), - $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ), - $r->{junk_change}, $total_junk, ( $jchp or 0 ); - } - else { - printf " + $r->{spam_change}, $th, ($r->{spam_change} / $th * 100), + $r->{spam_change}, $total_spam, ($schp or 0), + $r->{junk_change}, $th, ($r->{junk_change} / $th * 100), + $r->{junk_change}, $total_junk, ($jchp or 0); + } + else { + printf " Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%) Spam changes / total spam hits : %6d/%-6d (%7.3f%%) Junk changes (junk -> ham) : %6d/%-6d (%7.3f%%) Junk changes / total junk hits : %6d/%-6d (%7.3f%%) ", - $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ), - $r->{spam_change}, $total_spam, ( $schp or 0 ), - $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ), - $r->{junk_change}, $total_junk, ( $jchp or 0 ); + $r->{spam_change}, $th, ($r->{spam_change} / $th * 100), + $r->{spam_change}, $total_spam, ($schp or 0), + $r->{junk_change}, $th, ($r->{junk_change} / $th * 100), + $r->{junk_change}, $total_junk, ($jchp or 0); + } + } + else { + print ","; + JsonObjectElt("spam_change", $r->{spam_change}, "%.4f"); + print ","; + JsonObjectElt("junk_change", $r->{junk_change}, "%.4f"); + } } - } - if ($correlations) { - print "Correlations report:\n"; + if ($correlations) { + if (!$json) { + print "Correlations report:\n"; - while (my ($cs,$hits) = each %{$r->{corr}}) { - my $corr_prob = $hits / $total; - my $sym_prob = $r->{hits} / $total; - printf "Probability of %s when %s fires: %.3f\n", $s, $cs, ($corr_prob / $sym_prob); + while (my ($cs, $hits) = each %{$r->{corr}}) { + my $corr_prob = $hits / $total; + my $sym_prob = $r->{hits} / $total; + printf "Probability of %s when %s fires: %.3f\n", $s, $cs, + ($corr_prob / $sym_prob); + } + } + else { + print ","; + print "\"correllations\":{"; + + my $has_comma = 0; + while (my ($cs, $hits) = each %{$r->{corr}}) { + if ($has_comma) { + print ","; + } + else { + $has_comma = 1; + } + my $corr_prob = $hits / $total; + my $sym_prob = $r->{hits} / $total; + JsonObjectElt($cs, ($corr_prob / $sym_prob) ,"%.4f"); + } + + print "}"; + } } + + print "}" if $json; + } + else { + print "Symbol $s has not been met\n" if !$json; } + print '-' x 80 . "\n" if !$json; } - else { - print "Symbol $s has not been met\n"; - } - - print '-' x 80 . "\n"; } } -print " +sub Summary() { + if (!$json) { + print " === Summary ", '=' x 68, " Messages scanned: $total"; -printf " [ %s / %s ] + printf " [ %s / %s ] ", $timeStamp{'start'}, $timeStamp{'end'} - if defined $timeStamp{'start'}; -say ''; -printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} - for sort keys %action; -say ''; -printf "scan time min/avg/max = %.2f/%.2f/%.2f s + if defined $timeStamp{'start'}; + say ''; + printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} + for sort keys %action; + say ''; + printf "scan time min/avg/max = %.2f/%.2f/%.2f s ", $scanTime{'min'} / 1000, - ($total) ? $scanTime{'total'} / $total / 1000 : undef, - $scanTime{'max'} / 1000 - if exists $scanTime{'min'}; -say '=' x 80; + ($total) ? $scanTime{'total'} / $total / 1000 : undef, + $scanTime{'max'} / 1000 + if exists $scanTime{'min'}; + say '=' x 80; + } + else { + JsonObjectElt("total", $total, "%d"); + print ","; -exit; + if (defined $timeStamp{'start'}) { + JsonObjectElt("start", $timeStamp{'start'}); + print ","; + } + + if (defined $timeStamp{'end'}) { + JsonObjectElt("end", $timeStamp{'end'}); + print ","; + } + + print "\"actions\":{"; + + my $has_comma = 0; + foreach my $a (sort keys %action) { + if ($has_comma) { + print ","; + } + else { + $has_comma = 1; + } + JsonObjectElt($a, $action{$a}, "%d"); + } + print "},"; + } +} sub ProcessLog { my $ts_format = &log_time_format($rspamd_log); @@ -511,6 +625,7 @@ rspamd_stats [options] [--symbol=SYM1 [--symbol=SYM2...]] [--log file] --end ending time (newest) for log parsing --num-logs=integer number of recent logfiles to analyze (all files in the directory by default) --exclude-logs=integer number of latest logs to exclude (0 by default) + --json print json output instead of human readable --help brief help message --man full documentation -- 2.39.5