From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Sun, 13 Aug 2017 12:07:37 +0000 (+0100)
Subject: [Feature] Add json output for rspamd_stats
X-Git-Tag: 1.7.0~723
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3cd7353a9f96adba1f51ed85d4dd6096aa26ffbe;p=rspamd.git

[Feature] Add json output for rspamd_stats
---

diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl
index 95d04b414..46ad3a652 100755
--- a/utils/rspamd_stats.pl
+++ b/utils/rspamd_stats.pl
@@ -22,6 +22,7 @@ my $endTime;
 my $num_logs;
 my $exclude_logs = 0;
 my $man = 0;
+my $json = 0;
 my $help = 0;
 
 # Associate file extensions with decompressors
@@ -46,6 +47,7 @@ GetOptions(
   "end=s" => \$endTime,
   "num-logs|n=i" => \$num_logs,
   "exclude-logs|x=i" => \$exclude_logs,
+  "json|j" => \$json,
   "help|?" => \$help,
   "man" => \$man
 ) or pod2usage(2);
@@ -119,94 +121,206 @@ else {
 
 my $total_ham = $total - ($total_spam + $total_junk);
 
-if ($total > 0) {
-  while (my ($s, $r) = each(%sym_res)) {
-    if ($r->{hits} > 0) {
-      my $th = $r->{hits};
-      my $sh = $r->{spam_hits};
-      my $jh = $r->{junk_hits};
-      my $hh = $r->{hits} - $sh - $jh;
-      my $htp = $hh * 100.0 / $total_ham if $total_ham != 0;
-      my $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
-      my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
-
-      printf "%s   avg. weight %.3f, hits %d(%.3f%%):
+if ($json) {
+  print "{";
+  &Summary();
+  print '"symbols":{';
+  &SymbolsStat();
+  print "}}\n";
+}
+else {
+  &SymbolsStat();
+  &Summary();
+}
+
+exit;
+
+sub SymbolsStat() {
+  if ($total > 0) {
+    my $has_comma = 0;
+    while (my ($s, $r) = each(%sym_res)) {
+      if ($r->{hits} > 0) {
+        my $th = $r->{hits};
+        my $sh = $r->{spam_hits};
+        my $jh = $r->{junk_hits};
+        my $hh = $r->{hits} - $sh - $jh;
+        my $htp = $hh * 100.0 / $total_ham if $total_ham != 0;
+        my $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
+        my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
+
+        if ($json) {
+          if ($has_comma) {
+            print ",";
+          }
+          else {
+            $has_comma = 1;
+          }
+          print "\"$s\":{";
+          JsonObjectElt("avg_weight", $r->{'weight'},"%.4f");
+          print ",";
+          JsonObjectElt("hits", $th, "%d");
+          print ",";
+          JsonObjectElt("hits_percentage", $th/$total, "%.4f");
+          print ",";
+          JsonObjectElt("spam_hits", $sh, "%d");
+          print ",";
+          JsonObjectElt("spam_to_total", $sh/$th, "%.4f");
+          print ",";
+          JsonObjectElt("spam_percentage", $stp/100.0 || 0, "%.4f");
+          print ",";
+          JsonObjectElt("ham_hits", $hh, "%d");
+          print ",";
+          JsonObjectElt("ham_to_total", $hh/$th, "%.4f");
+          print ",";
+          JsonObjectElt("ham_percentage", $htp/100.0 || 0, "%.4f");
+          print ",";
+          JsonObjectElt("junk_hits", $jh, "%d");
+          print ",";
+          JsonObjectElt("junk_to_total", $jh/$th, "%.4f");
+          print ",";
+          JsonObjectElt("junk_percentage", $jtp/100.0 || 0, "%.4f");
+        }
+        else {
+          printf "%s   avg. weight %.3f, hits %d(%.3f%%):
   Ham  %7.3f%%, %6d/%-6d (%7.3f%%)
   Spam %7.3f%%, %6d/%-6d (%7.3f%%)
   Junk %7.3f%%, %6d/%-6d (%7.3f%%)
 ",
-        $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ),
-        ( $hh / $th * 100 ), $hh, $total_ham,  ( $htp or 0 ),
-        ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ),
-        ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 );
+            $s, $r->{weight} / $r->{hits}, $th, ($th / $total * 100),
+            ($hh / $th * 100), $hh, $total_ham, ($htp or 0),
+            ($sh / $th * 100), $sh, $total_spam, ($stp or 0),
+            ($jh / $th * 100), $jh, $total_junk, ($jtp or 0);
+        }
 
-      my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
-      my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
+        my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
+        my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
 
-      if ($r->{weight} != 0) {
-        if ($r->{weight} > 0) {
-          printf "
+        if ($r->{weight} != 0) {
+          if (!$json) {
+            if ($r->{weight} > 0) {
+              printf "
 Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%)
 Spam  changes / total spam hits: %6d/%-6d (%7.3f%%)
 Junk changes      (ham -> junk): %6d/%-6d (%7.3f%%)
 Junk  changes / total junk hits: %6d/%-6d (%7.3f%%)
 ",
-            $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
-            $r->{spam_change}, $total_spam, ( $schp or 0 ),
-            $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
-            $r->{junk_change}, $total_junk, ( $jchp or 0 );
-        }
-        else {
-          printf "
+                $r->{spam_change}, $th, ($r->{spam_change} / $th * 100),
+                $r->{spam_change}, $total_spam, ($schp or 0),
+                $r->{junk_change}, $th, ($r->{junk_change} / $th * 100),
+                $r->{junk_change}, $total_junk, ($jchp or 0);
+            }
+            else {
+              printf "
 Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%)
 Spam changes / total spam hits : %6d/%-6d (%7.3f%%)
 Junk changes (junk -> ham)     : %6d/%-6d (%7.3f%%)
 Junk changes / total junk hits : %6d/%-6d (%7.3f%%)
 ",
-            $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
-            $r->{spam_change}, $total_spam, ( $schp or 0 ),
-            $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
-            $r->{junk_change}, $total_junk, ( $jchp or 0 );
+                $r->{spam_change}, $th, ($r->{spam_change} / $th * 100),
+                $r->{spam_change}, $total_spam, ($schp or 0),
+                $r->{junk_change}, $th, ($r->{junk_change} / $th * 100),
+                $r->{junk_change}, $total_junk, ($jchp or 0);
+            }
+          }
+          else {
+            print ",";
+            JsonObjectElt("spam_change", $r->{spam_change}, "%.4f");
+            print ",";
+            JsonObjectElt("junk_change", $r->{junk_change}, "%.4f");
+          }
         }
-      }
 
-      if ($correlations) {
-        print "Correlations report:\n";
+        if ($correlations) {
+          if (!$json) {
+            print "Correlations report:\n";
 
-        while (my ($cs,$hits) = each %{$r->{corr}}) {
-          my $corr_prob = $hits / $total;
-          my $sym_prob = $r->{hits} / $total;
-          printf "Probability of %s when %s fires: %.3f\n", $s, $cs, ($corr_prob / $sym_prob);
+            while (my ($cs, $hits) = each %{$r->{corr}}) {
+              my $corr_prob = $hits / $total;
+              my $sym_prob = $r->{hits} / $total;
+              printf "Probability of %s when %s fires: %.3f\n", $s, $cs,
+                ($corr_prob / $sym_prob);
+            }
+          }
+          else {
+            print ",";
+            print "\"correllations\":{";
+
+            my $has_comma = 0;
+            while (my ($cs, $hits) = each %{$r->{corr}}) {
+              if ($has_comma) {
+                print ",";
+              }
+              else {
+                $has_comma = 1;
+              }
+              my $corr_prob = $hits / $total;
+              my $sym_prob = $r->{hits} / $total;
+              JsonObjectElt($cs, ($corr_prob / $sym_prob) ,"%.4f");
+            }
+
+            print "}";
+          }
         }
+
+        print "}" if $json;
+      }
+      else {
+        print "Symbol $s has not been met\n" if !$json;
       }
 
+      print '-' x 80 . "\n" if !$json;
     }
-    else {
-      print "Symbol $s has not been met\n";
-    }
-
-    print '-' x 80 . "\n";
   }
 }
 
-print "
+sub Summary() {
+  if (!$json) {
+    print "
 === Summary ", '=' x 68, "
 Messages scanned: $total";
-printf " [ %s / %s ]
+    printf " [ %s / %s ]
 ", $timeStamp{'start'}, $timeStamp{'end'}
-  if defined $timeStamp{'start'};
-say '';
-printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_}
-  for sort keys %action;
-say '';
-printf "scan time min/avg/max = %.2f/%.2f/%.2f s
+      if defined $timeStamp{'start'};
+    say '';
+    printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_}
+      for sort keys %action;
+    say '';
+    printf "scan time min/avg/max = %.2f/%.2f/%.2f s
 ", $scanTime{'min'} / 1000,
-  ($total) ? $scanTime{'total'} / $total / 1000 : undef,
-  $scanTime{'max'} / 1000
-  if exists $scanTime{'min'};
-say '=' x 80;
+        ($total) ? $scanTime{'total'} / $total / 1000 : undef,
+      $scanTime{'max'} / 1000
+      if exists $scanTime{'min'};
+    say '=' x 80;
+  }
+  else {
+    JsonObjectElt("total", $total, "%d");
+    print ",";
 
-exit;
+    if (defined $timeStamp{'start'}) {
+      JsonObjectElt("start", $timeStamp{'start'});
+      print ",";
+    }
+
+    if (defined $timeStamp{'end'}) {
+      JsonObjectElt("end", $timeStamp{'end'});
+      print ",";
+    }
+
+    print "\"actions\":{";
+
+    my $has_comma = 0;
+    foreach my $a (sort keys %action) {
+      if ($has_comma) {
+        print ",";
+      }
+      else {
+        $has_comma = 1;
+      }
+      JsonObjectElt($a, $action{$a}, "%d");
+    }
+    print "},";
+  }
+}
 
 sub ProcessLog {
   my $ts_format = &log_time_format($rspamd_log);
@@ -511,6 +625,7 @@ rspamd_stats [options] [--symbol=SYM1 [--symbol=SYM2...]] [--log file]
    --end                  ending time (newest) for log parsing
    --num-logs=integer     number of recent logfiles to analyze (all files in the directory by default)
    --exclude-logs=integer number of latest logs to exclude (0 by default)
+   --json                 print json output instead of human readable
    --help                 brief help message
    --man                  full documentation