$Net::MRT::USE_RFC4760 = -1;
my %config = (
- asn_sources => [
- 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
- 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
- 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
- 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
- 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
- ],
- bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
+ asn_sources => [
+ 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
+ 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
+ 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
+ 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
+ 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
+ ],
+ bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
);
my $download_asn = 0;
my $v6_zone = "asn6.rspamd.com";
my $v4_file = "asn.zone";
my $v6_file = "asn6.zone";
-my $ns_servers = ["asn-ns.rspamd.com", "asn-ns2.rspamd.com"];
+my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
GetOptions(
- "download-asn" => \$download_asn,
- "download-bgp" => \$download_bgp,
- "4!" => \$v4,
- "6!" => \$v6,
- "parse!" => \$parse,
- "target=s" => \$download_target,
- "zone-v4=s" => \$v4_zone,
- "zone-v6=s" => \$v6_zone,
- "file-v4=s" => \$v4_file,
- "file-v6=s" => \$v6_file,
- "ns-server=s@" => \$ns_servers,
- "help|?" => \$help,
- "man" => \$man
+ "download-asn" => \$download_asn,
+ "download-bgp" => \$download_bgp,
+ "4!" => \$v4,
+ "6!" => \$v6,
+ "parse!" => \$parse,
+ "target=s" => \$download_target,
+ "zone-v4=s" => \$v4_zone,
+ "zone-v6=s" => \$v6_zone,
+ "file-v4=s" => \$v4_file,
+ "file-v6=s" => \$v6_file,
+ "ns-server=s@" => \$ns_servers,
+ "help|?" => \$help,
+ "man" => \$man
) or pod2usage(2);
pod2usage(1) if $help;
pod2usage( -exitval => 0, -verbose => 2 ) if $man;
sub download_file {
- my ($u) = @_;
+ my ($u) = @_;
- print "Fetching $u\n";
- my $ff = File::Fetch->new( uri => $u );
- my $where = $ff->fetch( to => $download_target ) or die $ff->error;
+ print "Fetching $u\n";
+ my $ff = File::Fetch->new( uri => $u );
+ my $where = $ff->fetch( to => $download_target ) or die $ff->error;
- return $where;
+ return $where;
}
if ($download_asn) {
- foreach my $u ( @{ $config{'asn_sources'} } ) {
- download_file($u);
- }
+ foreach my $u ( @{ $config{'asn_sources'} } ) {
+ download_file($u);
+ }
}
if ($download_bgp) {
- foreach my $u ( @{ $config{'bgp_sources'} } ) {
- download_file($u);
- }
+ foreach my $u ( @{ $config{'bgp_sources'} } ) {
+ download_file($u);
+ }
}
if ( !$parse ) {
- exit 0;
+ exit 0;
}
my $v4_fh;
my $v6_fh;
if ($v4) {
- open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!";
- print $v4_fh
- "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
- foreach my $ns (@{$ns_servers}) {
- print $v4_fh "\$NS 43200 $ns\n";
- }
+ open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!";
+ print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
+ foreach my $ns ( @{$ns_servers} ) {
+ print $v4_fh "\$NS 43200 $ns\n";
+ }
}
if ($v6) {
- open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!";
- print $v6_fh
- "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
- foreach my $ns (@{$ns_servers}) {
- print $v6_fh "\$NS 43200 $ns\n";
- }
+ open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!";
+ print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
+ foreach my $ns ( @{$ns_servers} ) {
+ print $v6_fh "\$NS 43200 $ns\n";
+ }
}
# Now load BGP data
my $networks = {};
foreach my $u ( @{ $config{'bgp_sources'} } ) {
- my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename( $parsed->path );
- open( my $fh, "<:gzip", $fname )
- or die "Cannot open $fname: $!";
-
- while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) {
- if ( $dd->{'prefix'} && $dd->{'bits'} ) {
- next if $dd->{'subtype'} == 2 and !$v4;
- next if $dd->{'subtype'} == 4 and !$v6;
- my $entry = $dd->{'entries'}->[0];
- my $net = $dd->{'prefix'} . '/' . $dd->{'bits'};
- if ( $entry && $entry->{'AS_PATH'} ) {
- my $as = $entry->{'AS_PATH'}->[-1];
- if (ref($as) eq "ARRAY") {
- $as = @{$as}[0];
+ my $parsed = URI->new($u);
+ my $fname = $download_target . '/' . basename( $parsed->path );
+ open( my $fh, "<:gzip", $fname )
+ or die "Cannot open $fname: $!";
+
+ while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) {
+ if ( $dd->{'prefix'} && $dd->{'bits'} ) {
+ next if $dd->{'subtype'} == 2 and !$v4;
+ next if $dd->{'subtype'} == 4 and !$v6;
+ my $entry = $dd->{'entries'}->[0];
+ my $net = $dd->{'prefix'} . '/' . $dd->{'bits'};
+ if ( $entry && $entry->{'AS_PATH'} ) {
+ my $as = $entry->{'AS_PATH'}->[-1];
+ if ( ref($as) eq "ARRAY" ) {
+ $as = @{$as}[0];
+ }
+
+ if ( !$networks->{$as} ) {
+ if ( $dd->{'subtype'} == 2 ) {
+ $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] };
+ }
+ else {
+ $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] };
+ }
+ }
+ else {
+ if ( $dd->{'subtype'} == 2 ) {
+ push @{ $networks->{$as}->{'nets_v4'} }, $net;
+ }
+ else {
+ push @{ $networks->{$as}->{'nets_v6'} }, $net;
+ }
+ }
+ }
}
-
- if ( !$networks->{$as} ) {
- if ( $dd->{'subtype'} == 2 ) {
- $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] };
- }
- else {
- $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] };
- }
- }
- else {
- if ( $dd->{'subtype'} == 2 ) {
- push @{ $networks->{$as}->{'nets_v4'} }, $net;
- }
- else {
- push @{ $networks->{$as}->{'nets_v6'} }, $net;
- }
- }
- }
}
- }
}
# Now roughly detect countries
foreach my $u ( @{ $config{'asn_sources'} } ) {
- my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename( $parsed->path );
- open( my $fh, "<", $fname ) or die "Cannot open $fname: $!";
-
- while (<$fh>) {
- next if /^\#/;
- chomp;
- my @elts = split /\|/;
-
- if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) {
- my $as_start = int( $elts[3] );
- my $as_end = $as_start + int( $elts[4] );
-
- for ( my $as = $as_start ; $as < $as_end ; $as++ ) {
- my $real_as = $as;
-
- if (ref($as) eq "ARRAY") {
- $real_as = @{$as}[0];
+ my $parsed = URI->new($u);
+ my $fname = $download_target . '/' . basename( $parsed->path );
+ open( my $fh, "<", $fname ) or die "Cannot open $fname: $!";
+
+ while (<$fh>) {
+ next if /^\#/;
+ chomp;
+ my @elts = split /\|/;
+
+ if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) {
+ my $as_start = int( $elts[3] );
+ my $as_end = $as_start + int( $elts[4] );
+
+ for ( my $as = $as_start ; $as < $as_end ; $as++ ) {
+ my $real_as = $as;
+
+ if ( ref($as) eq "ARRAY" ) {
+ $real_as = @{$as}[0];
+ }
+
+ if ( $networks->{"$real_as"} ) {
+ $networks->{"$real_as"}->{'country'} = $elts[1];
+ $networks->{"$real_as"}->{'rir'} = $elts[0];
+ }
+ }
}
-
- if ( $networks->{"$real_as"} ) {
- $networks->{"$real_as"}->{'country'} = $elts[1];
- $networks->{"$real_as"}->{'rir'} = $elts[0];
- }
- }
}
- }
}
while ( my ( $k, $v ) = each( %{$networks} ) ) {
- if ($v4) {
- foreach my $n ( @{ $v->{'nets_v4'} } ) {
-
- # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
- if ( $v->{'country'} ) {
- printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
- }
- else {
- printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
- }
+ if ($v4) {
+ foreach my $n ( @{ $v->{'nets_v4'} } ) {
+
+ # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
+ if ( $v->{'country'} ) {
+ printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
+ }
+ else {
+ printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
+ }
+ }
}
- }
- if ($v6) {
- foreach my $n ( @{ $v->{'nets_v6'} } ) {
-
- # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
- if ( $v->{'country'} ) {
- printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
- }
- else {
- printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
- }
+ if ($v6) {
+ foreach my $n ( @{ $v->{'nets_v6'} } ) {
+
+ # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
+ if ( $v->{'country'} ) {
+ printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
+ }
+ else {
+ printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
+ }
+ }
}
- }
}
__END__
my $help;
GetOptions(
- "spam|s=s" => \$spam_dir,
- "ham|h=s" => \$ham_dir,
- "spam-symbol=s" => \$spam_symbol,
- "ham-symbol=s" => \$ham_symbol,
- "classifier|c=s" => \$classifier,
- "timeout|t=f" => \$timeout,
- "parallel|p=i" => \$parallel,
- "train-fraction|t=f" => \$train_fraction,
- "bogofilter|b" => \$use_bogofilter,
- "dspam|d" => \$use_dspam,
- "check-only" => \$check_only,
- "help|?" => \$help,
- "man" => \$man
+ "spam|s=s" => \$spam_dir,
+ "ham|h=s" => \$ham_dir,
+ "spam-symbol=s" => \$spam_symbol,
+ "ham-symbol=s" => \$ham_symbol,
+ "classifier|c=s" => \$classifier,
+ "timeout|t=f" => \$timeout,
+ "parallel|p=i" => \$parallel,
+ "train-fraction|t=f" => \$train_fraction,
+ "bogofilter|b" => \$use_bogofilter,
+ "dspam|d" => \$use_dspam,
+ "check-only" => \$check_only,
+ "help|?" => \$help,
+ "man" => \$man
) or pod2usage(2);
pod2usage(1) if $help;
pod2usage( -exitval => 0, -verbose => 2 ) if $man;
sub read_dir_files {
- my ( $dir, $target ) = @_;
- opendir( my $dh, $dir ) or die "cannot open dir $dir: $!";
- while ( my $file = readdir $dh ) {
- if ( -f "$dir/$file" ) {
- push @{$target}, "$dir/$file";
+ my ( $dir, $target ) = @_;
+ opendir( my $dh, $dir ) or die "cannot open dir $dir: $!";
+ while ( my $file = readdir $dh ) {
+ if ( -f "$dir/$file" ) {
+ push @{$target}, "$dir/$file";
+ }
}
- }
}
sub shuffle_array {
- my ($ar) = @_;
+ my ($ar) = @_;
- for ( my $i = 0 ; $i < scalar @{$ar} ; $i++ ) {
- if ( $i > 1 ) {
- my $sel = int( rand( $i - 1 ) );
- ( @{$ar}[$i], @{$ar}[$sel] ) = ( @{$ar}[$sel], @{$ar}[$i] );
+ for ( my $i = 0 ; $i < scalar @{$ar} ; $i++ ) {
+ if ( $i > 1 ) {
+ my $sel = int( rand( $i - 1 ) );
+ ( @{$ar}[$i], @{$ar}[$sel] ) = ( @{$ar}[$sel], @{$ar}[$i] );
+ }
}
- }
}
sub learn_rspamc {
- my ( $files, $spam ) = @_;
- my $processed = 0;
-
- my $cmd = $spam ? "learn_spam" : "learn_ham";
- my $args_quoted = shell_quote @{$files};
- open(
- my $p,
-"$rspamc -t $timeout -c $classifier --compact -j -n $parallel $cmd $args_quoted |"
- ) or die "cannot spawn $rspamc: $!";
-
- while (<$p>) {
- my $res = eval('decode_json($_)');
- if ( $res && $res->{'success'} ) {
- $processed++;
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ my $cmd = $spam ? "learn_spam" : "learn_ham";
+ my $args_quoted = shell_quote @{$files};
+ open( my $p, "$rspamc -t $timeout -c $classifier --compact -j -n $parallel $cmd $args_quoted |" )
+ or die "cannot spawn $rspamc: $!";
+
+ while (<$p>) {
+ my $res = eval('decode_json($_)');
+ if ( $res && $res->{'success'} ) {
+ $processed++;
+ }
}
- }
- return $processed;
+ return $processed;
}
sub learn_bogofilter {
- my ( $files, $spam ) = @_;
- my $processed = 0;
-
- foreach my $f ( @{$files} ) {
- my $args_quoted = shell_quote $f;
- my $fl = $spam ? "-s" : "-n";
- `$bogofilter -I $args_quoted $fl`;
- if ( $? == 0 ) {
- $processed++;
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+ my $fl = $spam ? "-s" : "-n";
+ `$bogofilter -I $args_quoted $fl`;
+ if ( $? == 0 ) {
+ $processed++;
+ }
}
- }
- return $processed;
+ return $processed;
}
sub learn_dspam {
- my ( $files, $spam ) = @_;
- my $processed = 0;
-
- foreach my $f ( @{$files} ) {
- my $args_quoted = shell_quote $f;
- my $fl = $spam ? "--class=spam" : "--class=innocent";
- open( my $p,
- "|$dspam --user nobody --source=corpus --stdout --mode=toe $fl" )
- or die "cannot run $dspam: $!";
-
- open( my $inp, "< $f" );
- while (<$inp>) {
- print $p $_;
+ my ( $files, $spam ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+ my $fl = $spam ? "--class=spam" : "--class=innocent";
+ open( my $p, "|$dspam --user nobody --source=corpus --stdout --mode=toe $fl" )
+ or die "cannot run $dspam: $!";
+
+ open( my $inp, "< $f" );
+ while (<$inp>) {
+ print $p $_;
+ }
}
- }
- return $processed;
+ return $processed;
}
sub learn_samples {
- my ( $ar_ham, $ar_spam ) = @_;
- my $len;
- my $processed = 0;
- my $total = 0;
- my $learn_func;
-
- my @files_spam;
- my @files_ham;
-
- if ($use_dspam) {
- $learn_func = \&learn_dspam;
- }
- elsif ($use_bogofilter) {
- $learn_func = \&learn_bogofilter;
- }
- else {
- $learn_func = \&learn_rspamc;
- }
-
- $len = int( scalar @{$ar_ham} * $train_fraction );
- my @cur_vec;
-
- # Shuffle spam and ham samples
- for ( my $i = 0 ; $i < $len ; $i++ ) {
- if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
- push @cur_vec, @{$ar_ham}[$i];
- push @files_ham, [@cur_vec];
- @cur_vec = ();
- $total++;
- }
- else {
- push @cur_vec, @{$ar_ham}[$i];
- }
- }
-
- $len = int( scalar @{$ar_spam} * $train_fraction );
- @cur_vec = ();
- for ( my $i = 0 ; $i < $len ; $i++ ) {
- if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
- push @cur_vec, @{$ar_spam}[$i];
- push @files_spam, [@cur_vec];
- @cur_vec = ();
- $total++;
- }
- else {
- push @cur_vec, @{$ar_spam}[$i];
+ my ( $ar_ham, $ar_spam ) = @_;
+ my $len;
+ my $processed = 0;
+ my $total = 0;
+ my $learn_func;
+
+ my @files_spam;
+ my @files_ham;
+
+ if ($use_dspam) {
+ $learn_func = \&learn_dspam;
}
- }
-
- for ( my $i = 0 ; $i < $total ; $i++ ) {
- my $args;
- my $spam;
-
- if ( $i % 2 == 0 ) {
- $args = pop @files_spam;
-
- if ( !$args ) {
- $args = pop @files_ham;
- $spam = 0;
- }
- else {
- $spam = 1;
- }
+ elsif ($use_bogofilter) {
+ $learn_func = \&learn_bogofilter;
}
else {
- $args = pop @files_ham;
- if ( !$args ) {
- $args = pop @files_spam;
- $spam = 1;
- }
- else {
- $spam = 0;
- }
+ $learn_func = \&learn_rspamc;
}
- my $r = $learn_func->( $args, $spam );
- if ($r) {
- $processed += $r;
+ $len = int( scalar @{$ar_ham} * $train_fraction );
+ my @cur_vec;
+
+ # Shuffle spam and ham samples
+ for ( my $i = 0 ; $i < $len ; $i++ ) {
+ if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
+ push @cur_vec, @{$ar_ham}[$i];
+ push @files_ham, [@cur_vec];
+ @cur_vec = ();
+ $total++;
+ }
+ else {
+ push @cur_vec, @{$ar_ham}[$i];
+ }
}
- }
- return $processed;
-}
+ $len = int( scalar @{$ar_spam} * $train_fraction );
+ @cur_vec = ();
+ for ( my $i = 0 ; $i < $len ; $i++ ) {
+ if ( $i > 0 && ( $i % $parallel == 0 || $i == $len - 1 ) ) {
+ push @cur_vec, @{$ar_spam}[$i];
+ push @files_spam, [@cur_vec];
+ @cur_vec = ();
+ $total++;
+ }
+ else {
+ push @cur_vec, @{$ar_spam}[$i];
+ }
+ }
-sub check_rspamc {
- my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+ for ( my $i = 0 ; $i < $total ; $i++ ) {
+ my $args;
+ my $spam;
- my $args_quoted = shell_quote @{$files};
- my $processed = 0;
+ if ( $i % 2 == 0 ) {
+ $args = pop @files_spam;
- open(
- my $p,
-"$rspamc -t $timeout -n $parallel --header=\"Settings: {symbols_enabled=[BAYES_SPAM]}\" --compact -j $args_quoted |"
- ) or die "cannot spawn $rspamc: $!";
-
- while (<$p>) {
- my $res = eval('decode_json($_)');
- if ( $res && $res->{'default'} ) {
- $processed++;
-
- if ($spam) {
- if ( $res->{'default'}->{$ham_symbol} ) {
- my $m = $res->{'default'}->{$ham_symbol}->{'options'}->[0];
- if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
- my $percentage = int($1);
- if ( $percentage >= $rspamc_prob_trigger ) {
- $$fp_cnt++;
+ if ( !$args ) {
+ $args = pop @files_ham;
+ $spam = 0;
+ }
+ else {
+ $spam = 1;
}
- }
- else {
- $$fp_cnt++;
- }
- }
- elsif ( !$res->{'default'}->{$spam_symbol} ) {
- $$fn_cnt++;
}
else {
- $$detected_cnt++;
- }
- }
- else {
- if ( $res->{'default'}->{$spam_symbol} ) {
- my $m = $res->{'default'}->{$spam_symbol}->{'options'}->[0];
- if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
-
- my $percentage = int($1);
- if ( $percentage >= $rspamc_prob_trigger ) {
- $$fp_cnt++;
+ $args = pop @files_ham;
+ if ( !$args ) {
+ $args = pop @files_spam;
+ $spam = 1;
+ }
+ else {
+ $spam = 0;
}
- }
- else {
- $$fp_cnt++;
- }
- }
- elsif ( !$res->{'default'}->{$ham_symbol} ) {
- $$fn_cnt++;
}
- else {
- $$detected_cnt++;
+
+ my $r = $learn_func->( $args, $spam );
+ if ($r) {
+ $processed += $r;
}
- }
}
- }
- return $processed;
+ return $processed;
}
-sub check_bogofilter {
- my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
- my $processed = 0;
+sub check_rspamc {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
- foreach my $f ( @{$files} ) {
- my $args_quoted = shell_quote $f;
+ my $args_quoted = shell_quote @{$files};
+ my $processed = 0;
- open( my $p, "$bogofilter -t -I $args_quoted |" )
- or die "cannot spawn $bogofilter: $!";
+ open(
+ my $p,
+"$rspamc -t $timeout -n $parallel --header=\"Settings: {symbols_enabled=[BAYES_SPAM]}\" --compact -j $args_quoted |"
+ ) or die "cannot spawn $rspamc: $!";
while (<$p>) {
- if ( $_ =~ /^([SHU])\s+.*$/ ) {
- $processed++;
-
- if ($spam) {
- if ( $1 eq 'H' ) {
- $$fp_cnt++;
- }
- elsif ( $1 eq 'U' ) {
- $$fn_cnt++;
- }
- else {
- $$detected_cnt++;
- }
- }
- else {
- if ( $1 eq 'S' ) {
- $$fp_cnt++;
- }
- elsif ( $1 eq 'U' ) {
- $$fn_cnt++;
- }
- else {
- $$detected_cnt++;
- }
+ my $res = eval('decode_json($_)');
+ if ( $res && $res->{'default'} ) {
+ $processed++;
+
+ if ($spam) {
+ if ( $res->{'default'}->{$ham_symbol} ) {
+ my $m = $res->{'default'}->{$ham_symbol}->{'options'}->[0];
+ if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
+ my $percentage = int($1);
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ else {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( !$res->{'default'}->{$spam_symbol} ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $res->{'default'}->{$spam_symbol} ) {
+ my $m = $res->{'default'}->{$spam_symbol}->{'options'}->[0];
+ if ( $m && $m =~ /^(\d+(?:\.\d+)?)%$/ ) {
+
+ my $percentage = int($1);
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ else {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( !$res->{'default'}->{$ham_symbol} ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
}
- }
}
- }
- return $processed;
+ return $processed;
}
-sub check_dspam {
- my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
- my $processed = 0;
+sub check_bogofilter {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+ my $processed = 0;
+
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
+
+ open( my $p, "$bogofilter -t -I $args_quoted |" )
+ or die "cannot spawn $bogofilter: $!";
+
+ while (<$p>) {
+ if ( $_ =~ /^([SHU])\s+.*$/ ) {
+ $processed++;
+
+ if ($spam) {
+ if ( $1 eq 'H' ) {
+ $$fp_cnt++;
+ }
+ elsif ( $1 eq 'U' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $1 eq 'S' ) {
+ $$fp_cnt++;
+ }
+ elsif ( $1 eq 'U' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ }
+ }
+ }
- foreach my $f ( @{$files} ) {
- my $args_quoted = shell_quote $f;
+ return $processed;
+}
- my $pid = open2( *Reader, *Writer,
- "$dspam --user nobody --classify --stdout --mode=notrain" );
- open( my $inp, "< $f" );
- while (<$inp>) {
- print Writer $_;
- }
- close Writer;
+sub check_dspam {
+ my ( $files, $spam, $fp_cnt, $fn_cnt, $detected_cnt ) = @_;
+ my $processed = 0;
- while (<Reader>) {
- if ( $_ =~
-qr(^X-DSPAM-Result: nobody; result="([^"]+)"; class="[^"]+"; probability=(\d+(?:\.\d+)?).*$)
- )
- {
- $processed++;
- my $percentage = int($2 * 100.0);
+ foreach my $f ( @{$files} ) {
+ my $args_quoted = shell_quote $f;
- if ($spam) {
- if ( $1 eq 'Innocent') {
- if ( $percentage <= (100 - $rspamc_prob_trigger) ) {
- $$fp_cnt++;
- }
- }
- elsif ( $1 ne 'Spam' ) {
- $$fn_cnt++;
- }
- else {
- $$detected_cnt++;
- }
+ my $pid = open2( *Reader, *Writer, "$dspam --user nobody --classify --stdout --mode=notrain" );
+ open( my $inp, "< $f" );
+ while (<$inp>) {
+ print Writer $_;
}
- else {
- if ( $1 eq 'Spam' ) {
- if ( $percentage >= $rspamc_prob_trigger ) {
- $$fp_cnt++;
+ close Writer;
+
+ while (<Reader>) {
+ if ( $_ =~ qr(^X-DSPAM-Result: nobody; result="([^"]+)"; class="[^"]+"; probability=(\d+(?:\.\d+)?).*$) ) {
+ $processed++;
+ my $percentage = int( $2 * 100.0 );
+
+ if ($spam) {
+ if ( $1 eq 'Innocent' ) {
+ if ( $percentage <= ( 100 - $rspamc_prob_trigger ) ) {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( $1 ne 'Spam' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
+ else {
+ if ( $1 eq 'Spam' ) {
+ if ( $percentage >= $rspamc_prob_trigger ) {
+ $$fp_cnt++;
+ }
+ }
+ elsif ( $1 ne 'Innocent' ) {
+ $$fn_cnt++;
+ }
+ else {
+ $$detected_cnt++;
+ }
+ }
}
- }
- elsif ( $1 ne 'Innocent' ) {
- $$fn_cnt++;
- }
- else {
- $$detected_cnt++;
- }
}
- }
+ close Reader;
+ waitpid( $pid, 0 );
}
- close Reader;
- waitpid( $pid, 0 );
- }
- return $processed;
+ return $processed;
}
sub cross_validate {
- my ($hr) = @_;
- my $args = "";
- my $processed = 0;
- my $fp_spam = 0;
- my $fn_spam = 0;
- my $fp_ham = 0;
- my $fn_ham = 0;
- my $total_spam = 0;
- my $total_ham = 0;
- my $detected_spam = 0;
- my $detected_ham = 0;
- my $i = 0;
- my $len = scalar keys %{$hr};
- my @files_spam;
- my @files_ham;
- my @cur_spam;
- my @cur_ham;
- my $check_func;
-
- if ($use_dspam) {
- $check_func = \&check_dspam;
- }
- elsif ($use_bogofilter) {
- $check_func = \&check_bogofilter;
- }
- else {
- $check_func = \&check_rspamc;
- }
-
- while ( my ( $fn, $spam ) = each( %{$hr} ) ) {
- if ($spam) {
- if ( scalar @cur_spam >= $parallel || $i == $len - 1 ) {
- push @cur_spam, $fn;
- push @files_spam, [@cur_spam];
- @cur_spam = ();
- }
- else {
- push @cur_spam, $fn;
- }
+ my ($hr) = @_;
+ my $args = "";
+ my $processed = 0;
+ my $fp_spam = 0;
+ my $fn_spam = 0;
+ my $fp_ham = 0;
+ my $fn_ham = 0;
+ my $total_spam = 0;
+ my $total_ham = 0;
+ my $detected_spam = 0;
+ my $detected_ham = 0;
+ my $i = 0;
+ my $len = scalar keys %{$hr};
+ my @files_spam;
+ my @files_ham;
+ my @cur_spam;
+ my @cur_ham;
+ my $check_func;
+
+ if ($use_dspam) {
+ $check_func = \&check_dspam;
+ }
+ elsif ($use_bogofilter) {
+ $check_func = \&check_bogofilter;
}
else {
- if ( scalar @cur_ham >= $parallel || $i == $len - 1 ) {
- push @cur_ham, $fn;
- push @files_ham, [@cur_ham];
- @cur_ham = ();
- }
- else {
- push @cur_ham, $fn;
- }
+ $check_func = \&check_rspamc;
+ }
+
+ while ( my ( $fn, $spam ) = each( %{$hr} ) ) {
+ if ($spam) {
+ if ( scalar @cur_spam >= $parallel || $i == $len - 1 ) {
+ push @cur_spam, $fn;
+ push @files_spam, [@cur_spam];
+ @cur_spam = ();
+ }
+ else {
+ push @cur_spam, $fn;
+ }
+ }
+ else {
+ if ( scalar @cur_ham >= $parallel || $i == $len - 1 ) {
+ push @cur_ham, $fn;
+ push @files_ham, [@cur_ham];
+ @cur_ham = ();
+ }
+ else {
+ push @cur_ham, $fn;
+ }
+ }
}
- }
- shuffle_array( \@files_spam );
+ shuffle_array( \@files_spam );
- foreach my $fn (@files_spam) {
- my $r = $check_func->( $fn, 1, \$fp_ham, \$fn_spam, \$detected_spam );
- $total_spam += $r;
- $processed += $r;
- }
+ foreach my $fn (@files_spam) {
+ my $r = $check_func->( $fn, 1, \$fp_ham, \$fn_spam, \$detected_spam );
+ $total_spam += $r;
+ $processed += $r;
+ }
- shuffle_array( \@files_ham );
+ shuffle_array( \@files_ham );
- foreach my $fn (@files_ham) {
- my $r = $check_func->( $fn, 0, \$fp_spam, \$fn_ham, \$detected_ham );
- $total_ham += $r;
- $processed += $r;
- }
+ foreach my $fn (@files_ham) {
+ my $r = $check_func->( $fn, 0, \$fp_spam, \$fn_ham, \$detected_ham );
+ $total_ham += $r;
+ $processed += $r;
+ }
- printf "Scanned %d messages
+ printf "Scanned %d messages
%d spam messages (%d detected)
-%d ham messages (%d detected)\n",
- $processed, $total_spam, $detected_spam, $total_ham, $detected_ham;
+%d ham messages (%d detected)\n", $processed, $total_spam, $detected_spam, $total_ham, $detected_ham;
- printf "\nHam FP rate: %.2f%% (%d messages)
-Ham FN rate: %.2f%% (%d messages)\n",
- $fp_ham / $total_ham * 100.0, $fp_ham,
- $fn_ham / $total_ham * 100.0, $fn_ham;
+ printf "\nHam FP rate: %.2f%% (%d messages)
+Ham FN rate: %.2f%% (%d messages)\n", $fp_ham / $total_ham * 100.0, $fp_ham, $fn_ham / $total_ham * 100.0, $fn_ham;
- printf "\nSpam FP rate: %.2f%% (%d messages)
+ printf "\nSpam FP rate: %.2f%% (%d messages)
Spam FN rate: %.2f%% (%d messages)\n",
- $fp_spam / $total_spam * 100.0, $fp_spam,
- $fn_spam / $total_spam * 100.0, $fn_spam;
+ $fp_spam / $total_spam * 100.0, $fp_spam,
+ $fn_spam / $total_spam * 100.0, $fn_spam;
}
if ( !$spam_dir || !$ham_dir ) {
- die "spam or/and ham directories are not specified";
+ die "spam or/and ham directories are not specified";
}
my @spam_samples;
shuffle_array( \@ham_samples );
if ( !$check_only ) {
- my $learned = 0;
- my $t0 = [gettimeofday];
- $learned = learn_samples( \@ham_samples, \@spam_samples );
- my $t1 = [gettimeofday];
+ my $learned = 0;
+ my $t0 = [gettimeofday];
+ $learned = learn_samples( \@ham_samples, \@spam_samples );
+ my $t1 = [gettimeofday];
- printf "Learned classifier, %d items processed, %.2f seconds elapsed\n",
- $learned, tv_interval( $t0, $t1 );
+ printf "Learned classifier, %d items processed, %.2f seconds elapsed\n", $learned, tv_interval( $t0, $t1 );
}
my %validation_set;
my $len = int( scalar @spam_samples * $train_fraction );
for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) {
- $validation_set{ $spam_samples[$i] } = 1;
+ $validation_set{ $spam_samples[$i] } = 1;
}
$len = int( scalar @ham_samples * $train_fraction );
for ( my $i = $len ; $i < scalar @spam_samples ; $i++ ) {
- $validation_set{ $ham_samples[$i] } = 0;
+ $validation_set{ $ham_samples[$i] } = 0;
}
cross_validate( \%validation_set );
my @symbols_groups;
my @symbols_ignored;
my %groups;
-my $reject_score = 15.0;
-my $junk_score = 6.0;
-my $diff_alpha = 0.1;
-my $correlations = 0;
-my $nrelated = 10;
-my $log_file = "";
+my $reject_score = 15.0;
+my $junk_score = 6.0;
+my $diff_alpha = 0.1;
+my $correlations = 0;
+my $nrelated = 10;
+my $log_file = "";
my $search_pattern = "";
-my $startTime="";
+my $startTime = "";
my $endTime;
my $num_logs;
my $exclude_logs = 0;
-my $man = 0;
-my $json = 0;
-my $help = 0;
+my $man = 0;
+my $json = 0;
+my $help = 0;
# Associate file extensions with decompressors
my %decompressor = (
);
GetOptions(
- "reject-score|r=f" => \$reject_score,
- "junk-score|j=f" => \$junk_score,
- "symbol|s=s@" => \@symbols_search,
- "symbol-bidir|S=s@" => \@symbols_bidirectional,
- "exclude|X=s@" => \@symbols_exclude,
- "ignore=s@" => \@symbols_ignored,
- "group|g=s@" => \@symbols_groups,
- "log|l=s" => \$log_file,
- "alpha-score|alpha|a=f" => \$diff_alpha,
- "correlations|c" => \$correlations,
- "nrelated=i" => \$nrelated,
- "search-pattern=s" => \$search_pattern,
- "start=s" => \$startTime,
- "end=s" => \$endTime,
- "num-logs|n=i" => \$num_logs,
- "exclude-logs|x=i" => \$exclude_logs,
- "json|j" => \$json,
- "help|?" => \$help,
- "man" => \$man
+ "reject-score|r=f" => \$reject_score,
+ "junk-score|j=f" => \$junk_score,
+ "symbol|s=s@" => \@symbols_search,
+ "symbol-bidir|S=s@" => \@symbols_bidirectional,
+ "exclude|X=s@" => \@symbols_exclude,
+ "ignore=s@" => \@symbols_ignored,
+ "group|g=s@" => \@symbols_groups,
+ "log|l=s" => \$log_file,
+ "alpha-score|alpha|a=f" => \$diff_alpha,
+ "correlations|c" => \$correlations,
+ "nrelated=i" => \$nrelated,
+ "search-pattern=s" => \$search_pattern,
+ "start=s" => \$startTime,
+ "end=s" => \$endTime,
+ "num-logs|n=i" => \$num_logs,
+ "exclude-logs|x=i" => \$exclude_logs,
+ "json|j" => \$json,
+ "help|?" => \$help,
+ "man" => \$man
) or pod2usage(2);
pod2usage(1) if $help;
-pod2usage(-exitval => 0, -verbose => 2) if $man;
-
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
# Global vars
-my $total = 0;
-my $total_spam = 0;
-my $total_junk = 0;
-my $junk_symbols = 0;
-my $spam_symbols = 0;
-my $ham_symbols = 0;
+my $total = 0;
+my $total_spam = 0;
+my $total_junk = 0;
+my $junk_symbols = 0;
+my $spam_symbols = 0;
+my $ham_symbols = 0;
my $ham_spam_change = 0;
my $ham_junk_change = 0;
my %sym_res;
my $rspamd_log;
-my $enabled = 0;
-my $log_file_num = 1;
+my $enabled = 0;
+my $log_file_num = 1;
my $spinner_update_time = 0;
my %action;
# Convert bidirectional symbols
foreach my $s (@symbols_bidirectional) {
- $bidir_match{$s} = {
- spam => "${s}_SPAM",
- ham => "${s}_HAM",
- };
- push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+ $bidir_match{$s} = {
+ spam => "${s}_SPAM",
+ ham => "${s}_HAM",
+ };
+ push @symbols_search, $s unless grep /^$s$/, @symbols_search;
}
# Deal with groups
my $group_id = 0;
foreach my $g (@symbols_groups) {
- my @symbols = split /,/,$g;
- my $group_name = "group$group_id";
+ my @symbols = split /,/, $g;
+ my $group_name = "group$group_id";
- foreach my $s (@symbols) {
- $groups{$s} = $group_name;
- push @symbols_search, $s unless grep /^$s$/, @symbols_search;
- }
+ foreach my $s (@symbols) {
+ $groups{$s} = $group_name;
+ push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+ }
}
@symbols_search = '.*'
unless @symbols_search;
-if ($log_file eq '-' || $log_file eq '') {
- $rspamd_log = \*STDIN;
- &ProcessLog();
+if ( $log_file eq '-' || $log_file eq '' ) {
+ $rspamd_log = \*STDIN;
+ &ProcessLog();
}
elsif ( -d "$log_file" ) {
- my $log_dir = "$log_file";
+ my $log_dir = "$log_file";
- my @logs = &GetLogfilesList($log_dir);
+ my @logs = &GetLogfilesList($log_dir);
- # Process logs
- foreach (@logs) {
- my $ext = (/[^.]+\.?([^.]*?)$/)[0];
- my $dc = $decompressor{$ext} || 'cat';
+ # Process logs
+ foreach (@logs) {
+ my $ext = (/[^.]+\.?([^.]*?)$/)[0];
+ my $dc = $decompressor{$ext} || 'cat';
- open( $rspamd_log, "-|", "$dc $log_dir/$_" )
- or die "cannot execute $dc $log_dir/$_ : $!";
+ open( $rspamd_log, "-|", "$dc $log_dir/$_" )
+ or die "cannot execute $dc $log_dir/$_ : $!";
- printf {interactive(*STDERR)} "\033[J Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs, $_;
- $spinner_update_time = 0; # Force spinner update
- &spinner;
+ printf { interactive(*STDERR) } "\033[J Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs,
+ $_;
+ $spinner_update_time = 0; # Force spinner update
+ &spinner;
- &ProcessLog;
+ &ProcessLog;
- close($rspamd_log)
- or warn "cannot close $dc $log_dir/$_: $!";
- }
- print {interactive(*STDERR)} "\033[J\033[G"; # Progress indicator clean-up
+ close($rspamd_log)
+ or warn "cannot close $dc $log_dir/$_: $!";
+ }
+ print { interactive(*STDERR) } "\033[J\033[G"; # Progress indicator clean-up
}
else {
- my $ext = ($log_file =~ /[^.]+\.?([^.]*?)$/)[0];
- my $dc = $decompressor{$ext} || 'cat';
- open( $rspamd_log, "-|", "$dc $log_file" )
- or die "cannot execute $dc $log_file : $!";
- $spinner_update_time = 0; # Force spinner update
- &spinner;
- &ProcessLog();
+ my $ext = ( $log_file =~ /[^.]+\.?([^.]*?)$/ )[0];
+ my $dc = $decompressor{$ext} || 'cat';
+ open( $rspamd_log, "-|", "$dc $log_file" )
+ or die "cannot execute $dc $log_file : $!";
+ $spinner_update_time = 0; # Force spinner update
+ &spinner;
+ &ProcessLog();
}
-my $total_ham = $total - ($total_spam + $total_junk);
+my $total_ham = $total - ( $total_spam + $total_junk );
if ($json) {
- print "{";
- &Summary();
- print '"symbols":{';
- &SymbolsStat();
- print "}}\n";
+ print "{";
+ &Summary();
+ print '"symbols":{';
+ &SymbolsStat();
+ print "}}\n";
}
else {
- &SymbolsStat();
- &Summary();
+ &SymbolsStat();
+ &Summary();
}
exit;
sub IsIgnored {
- my ($sym) = @_;
+ my ($sym) = @_;
- foreach my $ex (@symbols_ignored) {
- if ($sym =~ /^$ex$/) {
- return 1;
+ foreach my $ex (@symbols_ignored) {
+ if ( $sym =~ /^$ex$/ ) {
+ return 1;
+ }
}
- }
- return 0;
+ return 0;
}
sub GenRelated {
- my ($htb, $target_sym) = @_;
-
- my @result;
- my $i = 0;
- foreach my $sym (sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb}) {
- if ($sym ne $target_sym) {
- my @elt = ($sym, $htb->{$sym});
- push @result, \@elt;
- $i ++;
- }
+ my ( $htb, $target_sym ) = @_;
+
+ my @result;
+ my $i = 0;
+ foreach my $sym ( sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb} ) {
+ if ( $sym ne $target_sym ) {
+ my @elt = ( $sym, $htb->{$sym} );
+ push @result, \@elt;
+ $i++;
+ }
- last if $i > $nrelated;
- }
+ last if $i > $nrelated;
+ }
- return \@result;
+ return \@result;
}
sub StringifyRelated {
- my ($ar, $total) = @_;
- return join("\n", (map { sprintf "\t%s(%s: %.1f%%)",
- $_->[0], $_->[1], $_->[1] / ($total * 1.0) * 100.0 } @{$ar}));
+ my ( $ar, $total ) = @_;
+ return
+ join( "\n", ( map { sprintf "\t%s(%s: %.1f%%)", $_->[0], $_->[1], $_->[1] / ( $total * 1.0 ) * 100.0 } @{$ar} ) );
}
sub SymbolsStat {
- if ($total > 0) {
- my $has_comma = 0;
- while (my ($s, $r) = each(%sym_res)) {
- if ($r->{hits} > 0) {
- my $th = $r->{hits};
- my $sh = $r->{spam_hits};
- my $jh = $r->{junk_hits};
- my $hh = $r->{hits} - $sh - $jh;
- my $htp = $hh * 100.0 / $total_ham if $total_ham != 0;
- my $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
- my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
-
- if ($json) {
- if ($has_comma) {
- print ",";
- }
- else {
- $has_comma = 1;
- }
- print "\"$s\":{";
- JsonObjectElt("avg_weight", $r->{'weight'},"%.4f");
- print ",";
- JsonObjectElt("hits", $th, "%d");
- print ",";
- JsonObjectElt("hits_percentage", $th/$total, "%.4f");
- print ",";
- JsonObjectElt("spam_hits", $sh, "%d");
- print ",";
- JsonObjectElt("spam_to_total", $sh/$th, "%.4f");
- print ",";
- JsonObjectElt("spam_percentage", $stp/100.0 || 0, "%.4f");
- print ",";
- JsonObjectElt("ham_hits", $hh, "%d");
- print ",";
- JsonObjectElt("ham_to_total", $hh/$th, "%.4f");
- print ",";
- JsonObjectElt("ham_percentage", $htp/100.0 || 0, "%.4f");
- print ",";
- JsonObjectElt("junk_hits", $jh, "%d");
- print ",";
- JsonObjectElt("junk_to_total", $jh/$th, "%.4f");
- print ",";
- JsonObjectElt("junk_percentage", $jtp/100.0 || 0, "%.4f");
- }
- else {
- printf "%s avg. weight %.3f, hits %d(%.3f%%):
+ if ( $total > 0 ) {
+ my $has_comma = 0;
+ while ( my ( $s, $r ) = each(%sym_res) ) {
+ if ( $r->{hits} > 0 ) {
+ my $th = $r->{hits};
+ my $sh = $r->{spam_hits};
+ my $jh = $r->{junk_hits};
+ my $hh = $r->{hits} - $sh - $jh;
+ my $htp = $hh * 100.0 / $total_ham if $total_ham != 0;
+ my $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
+ my $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
+
+ if ($json) {
+ if ($has_comma) {
+ print ",";
+ }
+ else {
+ $has_comma = 1;
+ }
+ print "\"$s\":{";
+ JsonObjectElt( "avg_weight", $r->{'weight'}, "%.4f" );
+ print ",";
+ JsonObjectElt( "hits", $th, "%d" );
+ print ",";
+ JsonObjectElt( "hits_percentage", $th / $total, "%.4f" );
+ print ",";
+ JsonObjectElt( "spam_hits", $sh, "%d" );
+ print ",";
+ JsonObjectElt( "spam_to_total", $sh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "spam_percentage", $stp / 100.0 || 0, "%.4f" );
+ print ",";
+ JsonObjectElt( "ham_hits", $hh, "%d" );
+ print ",";
+ JsonObjectElt( "ham_to_total", $hh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "ham_percentage", $htp / 100.0 || 0, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_hits", $jh, "%d" );
+ print ",";
+ JsonObjectElt( "junk_to_total", $jh / $th, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_percentage", $jtp / 100.0 || 0, "%.4f" );
+ }
+ else {
+ printf "%s avg. weight %.3f, hits %d(%.3f%%):
Ham %7.3f%%, %6d/%-6d (%7.3f%%)
Spam %7.3f%%, %6d/%-6d (%7.3f%%)
Junk %7.3f%%, %6d/%-6d (%7.3f%%)
-",
- $s, $r->{weight} / $r->{hits}, $th, ($th / $total * 100),
- ($hh / $th * 100), $hh, $total_ham, ($htp or 0),
- ($sh / $th * 100), $sh, $total_spam, ($stp or 0),
- ($jh / $th * 100), $jh, $total_junk, ($jtp or 0);
- }
+", $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ),
+ ( $hh / $th * 100 ), $hh, $total_ham, ( $htp or 0 ),
+ ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ),
+ ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 );
+ }
- my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
- my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
+ my $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
+ my $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
- if ($r->{weight} != 0) {
- if (!$json) {
- if ($r->{weight} > 0) {
- printf "
+ if ( $r->{weight} != 0 ) {
+ if ( !$json ) {
+ if ( $r->{weight} > 0 ) {
+ printf "
Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%)
Spam changes / total spam hits: %6d/%-6d (%7.3f%%)
Junk changes (ham -> junk): %6d/%-6d (%7.3f%%)
Junk changes / total junk hits: %6d/%-6d (%7.3f%%)
",
- $r->{spam_change}, $th, ($r->{spam_change} / $th * 100),
- $r->{spam_change}, $total_spam, ($schp or 0),
- $r->{junk_change}, $th, ($r->{junk_change} / $th * 100),
- $r->{junk_change}, $total_junk, ($jchp or 0);
- }
- else {
- printf "
+ $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
+ $r->{spam_change}, $total_spam, ( $schp or 0 ),
+ $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
+ $r->{junk_change}, $total_junk, ( $jchp or 0 );
+ }
+ else {
+ printf "
Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%)
Spam changes / total spam hits : %6d/%-6d (%7.3f%%)
Junk changes (junk -> ham) : %6d/%-6d (%7.3f%%)
Junk changes / total junk hits : %6d/%-6d (%7.3f%%)
",
- $r->{spam_change}, $th, ($r->{spam_change} / $th * 100),
- $r->{spam_change}, $total_spam, ($schp or 0),
- $r->{junk_change}, $th, ($r->{junk_change} / $th * 100),
- $r->{junk_change}, $total_junk, ($jchp or 0);
- }
- }
- else {
- print ",";
- JsonObjectElt("spam_change", $r->{spam_change}, "%.4f");
- print ",";
- JsonObjectElt("junk_change", $r->{junk_change}, "%.4f");
- }
- }
-
- if ($correlations) {
-
- my $spam_related = GenRelated($r->{symbols_met_spam}, $s);
- my $junk_related = GenRelated($r->{symbols_met_junk}, $s);
- my $ham_related = GenRelated($r->{symbols_met_ham}, $s);
-
- if (!$json) {
- print "Correlations report:\n";
-
- while (my ($cs, $hits) = each %{$r->{corr}}) {
- my $corr_prob = $r->{'hits'} / $total;
- my $merged_hits = 0;
- if($r->{symbols_met_spam}->{$cs}) {
- $merged_hits += $r->{symbols_met_spam}->{$cs};
- }
- if($r->{symbols_met_junk}->{$cs}) {
- $merged_hits += $r->{symbols_met_junk}->{$cs};
- }
- if($r->{symbols_met_ham}->{$cs}) {
- $merged_hits += $r->{symbols_met_ham}->{$cs};
- }
-
- if ($merged_hits > 0) {
- printf "Probability of %s when %s fires: %.3f\n", $cs, $s,
- (($merged_hits / $total) / $corr_prob);
- }
- }
+ $r->{spam_change}, $th, ( $r->{spam_change} / $th * 100 ),
+ $r->{spam_change}, $total_spam, ( $schp or 0 ),
+ $r->{junk_change}, $th, ( $r->{junk_change} / $th * 100 ),
+ $r->{junk_change}, $total_junk, ( $jchp or 0 );
+ }
+ }
+ else {
+ print ",";
+ JsonObjectElt( "spam_change", $r->{spam_change}, "%.4f" );
+ print ",";
+ JsonObjectElt( "junk_change", $r->{junk_change}, "%.4f" );
+ }
+ }
- print "Related symbols report:\n";
- printf "Top related in spam:\n %s\n", StringifyRelated($spam_related,
- $r->{spam_hits});
- printf "Top related in junk:\n %s\n", StringifyRelated($junk_related,
- $r->{junk_hits});
- printf "Top related in ham:\n %s\n", StringifyRelated($ham_related,
- $r->{hits} - $r->{spam_hits} - $r->{junk_hits});
- }
- else {
- print ",";
- print "\"correllations\":{";
+ if ($correlations) {
+
+ my $spam_related = GenRelated( $r->{symbols_met_spam}, $s );
+ my $junk_related = GenRelated( $r->{symbols_met_junk}, $s );
+ my $ham_related = GenRelated( $r->{symbols_met_ham}, $s );
+
+ if ( !$json ) {
+ print "Correlations report:\n";
+
+ while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+ my $corr_prob = $r->{'hits'} / $total;
+ my $merged_hits = 0;
+ if ( $r->{symbols_met_spam}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_spam}->{$cs};
+ }
+ if ( $r->{symbols_met_junk}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_junk}->{$cs};
+ }
+ if ( $r->{symbols_met_ham}->{$cs} ) {
+ $merged_hits += $r->{symbols_met_ham}->{$cs};
+ }
+
+ if ( $merged_hits > 0 ) {
+ printf "Probability of %s when %s fires: %.3f\n", $cs, $s,
+ ( ( $merged_hits / $total ) / $corr_prob );
+ }
+ }
+
+ print "Related symbols report:\n";
+ printf "Top related in spam:\n %s\n", StringifyRelated( $spam_related, $r->{spam_hits} );
+ printf "Top related in junk:\n %s\n", StringifyRelated( $junk_related, $r->{junk_hits} );
+ printf "Top related in ham:\n %s\n",
+ StringifyRelated( $ham_related, $r->{hits} - $r->{spam_hits} - $r->{junk_hits} );
+ }
+ else {
+ print ",";
+ print "\"correllations\":{";
+
+ my $has_comma_ = 0;
+ while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+ if ($has_comma_) {
+ print ",";
+ }
+ else {
+ $has_comma_ = 1;
+ }
+ my $corr_prob = $hits / $total;
+ my $sym_prob = $r->{hits} / $total;
+ JsonObjectElt( $cs, ( $corr_prob / $sym_prob ), "%.4f" );
+ }
+
+ print "}";
+ }
+ }
- my $has_comma_ = 0;
- while (my ($cs, $hits) = each %{$r->{corr}}) {
- if ($has_comma_) {
- print ",";
- }
- else {
- $has_comma_ = 1;
- }
- my $corr_prob = $hits / $total;
- my $sym_prob = $r->{hits} / $total;
- JsonObjectElt($cs, ($corr_prob / $sym_prob) ,"%.4f");
+ print "}" if $json;
+ }
+ else {
+ print "Symbol $s has not been met\n" if !$json;
}
- print "}";
- }
+ print '-' x 80 . "\n" if !$json;
}
-
- print "}" if $json;
- }
- else {
- print "Symbol $s has not been met\n" if !$json;
- }
-
- print '-' x 80 . "\n" if !$json;
}
- }
}
sub Summary() {
- if (!$json) {
- print "
+ if ( !$json ) {
+ print "
=== Summary ", '=' x 68, "
Messages scanned: $total";
- printf " [ %s / %s ]
+ printf " [ %s / %s ]
", $timeStamp{'start'}, $timeStamp{'end'}
- if defined $timeStamp{'start'};
- say '';
- printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_}
- for sort keys %action;
- say '';
- printf "scan time min/avg/max = %.2f/%.2f/%.2f s
-", $scanTime{'min'} / 1000,
- ($total) ? $scanTime{'total'} / $total / 1000 : undef,
- $scanTime{'max'} / 1000
- if exists $scanTime{'min'};
- say '=' x 80;
- }
- else {
- JsonObjectElt("total", $total, "%d");
- print ",";
-
- if (defined $timeStamp{'start'}) {
- JsonObjectElt("start", $timeStamp{'start'});
- print ",";
+ if defined $timeStamp{'start'};
+ say '';
+ printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} for sort keys %action;
+ say '';
+ printf "scan time min/avg/max = %.2f/%.2f/%.2f s
+", $scanTime{'min'} / 1000, ($total) ? $scanTime{'total'} / $total / 1000 : undef, $scanTime{'max'} / 1000
+ if exists $scanTime{'min'};
+ say '=' x 80;
}
+ else {
+ JsonObjectElt( "total", $total, "%d" );
+ print ",";
- if (defined $timeStamp{'end'}) {
- JsonObjectElt("end", $timeStamp{'end'});
- print ",";
- }
+ if ( defined $timeStamp{'start'} ) {
+ JsonObjectElt( "start", $timeStamp{'start'} );
+ print ",";
+ }
- print "\"actions\":{";
+ if ( defined $timeStamp{'end'} ) {
+ JsonObjectElt( "end", $timeStamp{'end'} );
+ print ",";
+ }
- my $has_comma = 0;
- foreach my $a (sort keys %action) {
- if ($has_comma) {
- print ",";
- }
- else {
- $has_comma = 1;
- }
- JsonObjectElt($a, $action{$a}, "%d");
+ print "\"actions\":{";
+
+ my $has_comma = 0;
+ foreach my $a ( sort keys %action ) {
+ if ($has_comma) {
+ print ",";
+ }
+ else {
+ $has_comma = 1;
+ }
+ JsonObjectElt( $a, $action{$a}, "%d" );
+ }
+ print "},";
}
- print "},";
- }
}
sub ProcessRelated {
- my ($symbols, $target, $source) = @_;
+ my ( $symbols, $target, $source ) = @_;
- foreach my $s (@{$symbols}) {
- $s =~ /^([^\(]+)(\(([^\)]+)\))?/;
- my $sym_name = $1;
- my $sym_score = 0;
+ foreach my $s ( @{$symbols} ) {
+ $s =~ /^([^\(]+)(\(([^\)]+)\))?/;
+ my $sym_name = $1;
+ my $sym_score = 0;
- if ($groups{$sym_name}) {
- $sym_name = $groups{$sym_name};
- }
+ if ( $groups{$sym_name} ) {
+ $sym_name = $groups{$sym_name};
+ }
- next if ($source eq $sym_name);
+ next if ( $source eq $sym_name );
- next if IsIgnored($sym_name);
+ next if IsIgnored($sym_name);
- if ($2) {
- $sym_score = $3 * 1.0;
+ if ($2) {
+ $sym_score = $3 * 1.0;
- if (abs($sym_score) < $diff_alpha) {
- next;
- }
+ if ( abs($sym_score) < $diff_alpha ) {
+ next;
+ }
+
+ my $bm = $bidir_match{$sym_name};
+ if ($bm) {
+ if ( $sym_score >= 0 ) {
+ $sym_name = $bm->{'spam'};
+ }
+ else {
+ $sym_name = $bm->{'ham'};
+ }
+ }
+ }
- my $bm = $bidir_match{$sym_name};
- if ($bm) {
- if ($sym_score >= 0) {
- $sym_name = $bm->{'spam'};
+ if ( exists( $target->{$sym_name} ) ) {
+ $target->{$sym_name}++;
}
else {
- $sym_name = $bm->{'ham'};
+ $target->{$sym_name} = 1;
}
- }
- }
-
- if (exists($target->{$sym_name})) {
- $target->{$sym_name} ++;
- }
- else {
- $target->{$sym_name} = 1;
}
- }
}
sub ProcessLog {
- my ( $ts_format, @line ) = &log_time_format($rspamd_log);
+ my ( $ts_format, @line ) = &log_time_format($rspamd_log);
- while() {
- last if eof $rspamd_log;
- $_ = (@line) ? shift @line : <$rspamd_log>;
+ while () {
+ last if eof $rspamd_log;
+ $_ = (@line) ? shift @line : <$rspamd_log>;
- if (!$enabled && ($search_pattern eq "" || /$search_pattern/)) {
- $enabled = 1;
- }
-
- next if !$enabled;
-
- if (/^.*rspamd_task_write_log.*$/) {
- &spinner;
- my $ts;
- if ( $ts_format eq 'syslog' ) {
- $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] );
- } elsif ( $ts_format eq 'syslog5424' ) {
- /^([0-9-]+)T([0-9:]+)/;
- $ts = "$1 $2";
- } else {
- $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ];
- }
-
- next if ( $ts lt $startTime );
- next if ( defined $endTime && $ts gt $endTime );
-
- if ($_ !~ /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms real/) {
- #print "BAD: $_\n";
- next;
- }
-
- my @symbols = split /(?:\{[^}]*\})?(?:$|,)/, $4;
- my $scan_time = $5;
- my $act = $1;
- my $score = $2 * 1.0;
- my $skip = 0;
-
- foreach my $ex (@symbols_exclude) {
- my @found = grep {/^$ex/} @symbols;
-
- if (scalar(@found) > 0) {
- $skip = 1;
- last;
+ if ( !$enabled && ( $search_pattern eq "" || /$search_pattern/ ) ) {
+ $enabled = 1;
}
- }
-
- next if ( $skip != 0 );
-
- if (defined($timeStamp{'end'})) {
- $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} );
- }
- else {
- $timeStamp{'end'} = $ts;
- }
-
- if (defined($timeStamp{'start'})) {
- $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} );
- }
- else {
- $timeStamp{'start'} = $ts;
- }
-
- $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time );
- $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time );
- $scanTime{'total'} += $scan_time;
-
- $action{$act}++;
- $total ++;
- if ($score >= $reject_score) {
- $total_spam ++;
- }
- elsif ($score >= $junk_score) {
- $total_junk ++;
- }
+ next if !$enabled;
- my @sym_names;
-
- foreach my $s (@symbols_search) {
- my @selected = grep /$s/, @symbols;
+ if (/^.*rspamd_task_write_log.*$/) {
+ &spinner;
+ my $ts;
+ if ( $ts_format eq 'syslog' ) {
+ $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] );
+ }
+ elsif ( $ts_format eq 'syslog5424' ) {
+ /^([0-9-]+)T([0-9:]+)/;
+ $ts = "$1 $2";
+ }
+ else {
+ $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ];
+ }
- if (scalar(@selected) > 0) {
+ next if ( $ts lt $startTime );
+ next if ( defined $endTime && $ts gt $endTime );
- foreach my $sym (@selected) {
- $sym =~ /^([^\(]+)(\(([^\)]+)\))?/;
- my $sym_name = $1;
- my $sym_score = 0;
- my $orig_name = $sym_name;
+ if ( $_ !~
+ /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms real/ )
+ {
+ #print "BAD: $_\n";
+ next;
+ }
- if ($2) {
- $sym_score = $3 * 1.0;
+ my @symbols = split /(?:\{[^}]*\})?(?:$|,)/, $4;
+ my $scan_time = $5;
+ my $act = $1;
+ my $score = $2 * 1.0;
+ my $skip = 0;
- if (abs($sym_score) < $diff_alpha) {
- next;
- }
+ foreach my $ex (@symbols_exclude) {
+ my @found = grep { /^$ex/ } @symbols;
- my $bm = $bidir_match{$sym_name};
- if ($bm) {
- if ($sym_score >= 0) {
- $sym_name = $bm->{'spam'};
- }
- else {
- $sym_name = $bm->{'ham'};
+ if ( scalar(@found) > 0 ) {
+ $skip = 1;
+ last;
}
- }
}
- next if $orig_name !~ /^$s/;
+ next if ( $skip != 0 );
- if ($groups{$s}) {
- # Replace with group
- $sym_name = $groups{$s};
+ if ( defined( $timeStamp{'end'} ) ) {
+ $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} );
+ }
+ else {
+ $timeStamp{'end'} = $ts;
}
- push @sym_names, $sym_name;
-
- if (!$sym_res{$sym_name}) {
- $sym_res{$sym_name} = {
- hits => 0,
- spam_hits => 0,
- junk_hits => 0,
- spam_change => 0,
- junk_change => 0,
- weight => 0,
- corr => {},
- symbols_met_spam => {},
- symbols_met_ham => {},
- symbols_met_junk => {},
- };
+ if ( defined( $timeStamp{'start'} ) ) {
+ $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} );
+ }
+ else {
+ $timeStamp{'start'} = $ts;
}
- my $r = $sym_res{$sym_name};
+ $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time );
+ $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time );
+ $scanTime{'total'} += $scan_time;
- $r->{hits} ++;
- $r->{weight} += $sym_score;
- my $is_spam = 0;
- my $is_junk = 0;
+ $action{$act}++;
+ $total++;
- if ($score >= $reject_score) {
- $is_spam = 1;
- $r->{spam_hits} ++;
- if ($correlations) {
- ProcessRelated(\@symbols, $r->{symbols_met_spam}, $sym_name);
- }
+ if ( $score >= $reject_score ) {
+ $total_spam++;
}
- elsif ($score >= $junk_score) {
- $is_junk = 1;
- $r->{junk_hits} ++;
- if ($correlations) {
- ProcessRelated(\@symbols, $r->{symbols_met_junk}, $sym_name);
- }
- }
- else {
- if ($correlations) {
- ProcessRelated(\@symbols, $r->{symbols_met_ham}, $sym_name);
- }
+ elsif ( $score >= $junk_score ) {
+ $total_junk++;
}
- if ($sym_score != 0) {
- my $score_without = $score - $sym_score;
-
- if ($sym_score > 0) {
- if ($is_spam && $score_without < $reject_score) {
- $r->{spam_change} ++;
- }
- if ($is_junk && $score_without < $junk_score) {
- $r->{junk_change} ++;
+ my @sym_names;
+
+ foreach my $s (@symbols_search) {
+ my @selected = grep /$s/, @symbols;
+
+ if ( scalar(@selected) > 0 ) {
+
+ foreach my $sym (@selected) {
+ $sym =~ /^([^\(]+)(\(([^\)]+)\))?/;
+ my $sym_name = $1;
+ my $sym_score = 0;
+ my $orig_name = $sym_name;
+
+ if ($2) {
+ $sym_score = $3 * 1.0;
+
+ if ( abs($sym_score) < $diff_alpha ) {
+ next;
+ }
+
+ my $bm = $bidir_match{$sym_name};
+ if ($bm) {
+ if ( $sym_score >= 0 ) {
+ $sym_name = $bm->{'spam'};
+ }
+ else {
+ $sym_name = $bm->{'ham'};
+ }
+ }
+ }
+
+ next if $orig_name !~ /^$s/;
+
+ if ( $groups{$s} ) {
+
+ # Replace with group
+ $sym_name = $groups{$s};
+ }
+
+ push @sym_names, $sym_name;
+
+ if ( !$sym_res{$sym_name} ) {
+ $sym_res{$sym_name} = {
+ hits => 0,
+ spam_hits => 0,
+ junk_hits => 0,
+ spam_change => 0,
+ junk_change => 0,
+ weight => 0,
+ corr => {},
+ symbols_met_spam => {},
+ symbols_met_ham => {},
+ symbols_met_junk => {},
+ };
+ }
+
+ my $r = $sym_res{$sym_name};
+
+ $r->{hits}++;
+ $r->{weight} += $sym_score;
+ my $is_spam = 0;
+ my $is_junk = 0;
+
+ if ( $score >= $reject_score ) {
+ $is_spam = 1;
+ $r->{spam_hits}++;
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_spam}, $sym_name );
+ }
+ }
+ elsif ( $score >= $junk_score ) {
+ $is_junk = 1;
+ $r->{junk_hits}++;
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_junk}, $sym_name );
+ }
+ }
+ else {
+ if ($correlations) {
+ ProcessRelated( \@symbols, $r->{symbols_met_ham}, $sym_name );
+ }
+ }
+
+ if ( $sym_score != 0 ) {
+ my $score_without = $score - $sym_score;
+
+ if ( $sym_score > 0 ) {
+ if ( $is_spam && $score_without < $reject_score ) {
+ $r->{spam_change}++;
+ }
+ if ( $is_junk && $score_without < $junk_score ) {
+ $r->{junk_change}++;
+ }
+ }
+ else {
+ if ( !$is_spam && $score_without >= $reject_score ) {
+ $r->{spam_change}++;
+ }
+ if ( !$is_junk && $score_without >= $junk_score ) {
+ $r->{junk_change}++;
+ }
+ }
+ }
+ } # End foreach symbols selected
}
- }
- else {
- if (!$is_spam && $score_without >= $reject_score) {
- $r->{spam_change} ++;
- }
- if (!$is_junk && $score_without >= $junk_score) {
- $r->{junk_change} ++;
- }
- }
}
- } # End foreach symbols selected
- }
- }
-
- if ($correlations) {
- foreach my $sym (@sym_names) {
- next if IsIgnored($sym);
- my $r = $sym_res{$sym};
-
- foreach my $corr_sym (@sym_names) {
- if ($corr_sym ne $sym) {
- if ($r->{'corr'}->{$corr_sym}) {
- $r->{'corr'}->{$corr_sym} ++;
- }
- else {
- $r->{'corr'}->{$corr_sym} = 1;
- }
+
+ if ($correlations) {
+ foreach my $sym (@sym_names) {
+ next if IsIgnored($sym);
+ my $r = $sym_res{$sym};
+
+ foreach my $corr_sym (@sym_names) {
+ if ( $corr_sym ne $sym ) {
+ if ( $r->{'corr'}->{$corr_sym} ) {
+ $r->{'corr'}->{$corr_sym}++;
+ }
+ else {
+ $r->{'corr'}->{$corr_sym} = 1;
+ }
+ }
+ }
+ } # End of correlations check
}
- }
- } # End of correlations check
- }
+ }
}
- }
}
sub JsonObjectElt() {
- my ($k, $v) = @_;
- my $f = defined $_[2] ? $_[2] : '%s';
+ my ( $k, $v ) = @_;
+ my $f = defined $_[2] ? $_[2] : '%s';
- if ($f eq "%s") {
- $f = "\"%s\"";
- }
+ if ( $f eq "%s" ) {
+ $f = "\"%s\"";
+ }
- printf "\"%s\":$f", $k, $v;
+ printf "\"%s\":$f", $k, $v;
}
sub GetLogfilesList {
- my ($dir) = @_;
- opendir( DIR, $dir ) or die $!;
+ my ($dir) = @_;
+ opendir( DIR, $dir ) or die $!;
- my $pattern = join( '|', keys %decompressor );
- my $re = qr/\.[0-9]+(?:\.(?:$pattern))?/;
+ my $pattern = join( '|', keys %decompressor );
+ my $re = qr/\.[0-9]+(?:\.(?:$pattern))?/;
- # Add unnumbered logs first
- my @logs =
- grep { -f "$dir/$_" && !/$re/ } readdir(DIR);
+ # Add unnumbered logs first
+ my @logs =
+ grep { -f "$dir/$_" && !/$re/ } readdir(DIR);
- # Add numbered logs
- rewinddir(DIR);
- push( @logs,
- ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir(DIR) ) ) );
+ # Add numbered logs
+ rewinddir(DIR);
+ push( @logs, ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir(DIR) ) ) );
- closedir(DIR);
+ closedir(DIR);
- # Select required logs and revers their order
- @logs =
- reverse
- splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs );
+ # Select required logs and revers their order
+ @logs =
+ reverse splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs );
- # Loop through array printing out filenames
- print {interactive(*STDERR)} "\nLog files to process:\n";
- foreach my $file (@logs) {
- print {interactive(*STDERR)} " $file\n";
- }
- print {interactive(*STDERR)} "\n";
+ # Loop through array printing out filenames
+ print { interactive(*STDERR) } "\nLog files to process:\n";
+ foreach my $file (@logs) {
+ print { interactive(*STDERR) } " $file\n";
+ }
+ print { interactive(*STDERR) } "\n";
- return @logs;
+ return @logs;
}
sub log_time_format {
- my $fh = shift;
- my ( $format, $line );
- while (<$fh>) {
- $line = $_;
-
- # 2017-08-08 00:00:01 #66984(
- # 2017-08-08 00:00:01.001 #66984(
- if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3})? #\d+\(/) {
- $format = 'rspamd';
- last;
- }
+ my $fh = shift;
+ my ( $format, $line );
+ while (<$fh>) {
+ $line = $_;
+
+ # 2017-08-08 00:00:01 #66984(
+ # 2017-08-08 00:00:01.001 #66984(
+ if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3})? #\d+\(/) {
+ $format = 'rspamd';
+ last;
+ }
- # Aug 8 00:02:50 #66986(
- elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) {
- $format = 'syslog';
- last;
- }
+ # Aug 8 00:02:50 #66986(
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) {
+ $format = 'syslog';
+ last;
+ }
- # Aug 8 00:02:50 hostname rspamd[66986]
- elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) {
- $format = 'syslog';
- last;
- }
+ # Aug 8 00:02:50 hostname rspamd[66986]
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) {
+ $format = 'syslog';
+ last;
+ }
- # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968]
- elsif(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) {
- $format = 'syslog5424';
- last;
- }
+ # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968]
+ elsif (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) {
+ $format = 'syslog5424';
+ last;
+ }
- # Skip newsyslog messages
- # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over
- elsif ( /^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/ ) {
- next;
- }
- # Skip journalctl messages
- # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. --
- elsif ( /^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/ ) {
- next;
- }
- else {
- print "Unknown log format\n";
- exit 1;
+ # Skip newsyslog messages
+ # Aug 8 00:00:00 hostname newsyslog[63284]: logfile turned over
+ elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/) {
+ next;
+ }
+
+ # Skip journalctl messages
+ # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. --
+ elsif (
+/^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/
+ )
+ {
+ next;
+ }
+ else {
+ print "Unknown log format\n";
+ exit 1;
+ }
}
- }
- return ( $format, $line );
+ return ( $format, $line );
}
sub normalized_time {
- return undef
- if !defined( $_ = shift );
+ return undef
+ if !defined( $_ = shift );
- /^\d\d(?::\d\d){0,2}$/
- ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4],
- (localtime)[3], $_
- : $_;
+ /^\d\d(?::\d\d){0,2}$/
+ ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4], (localtime)[3], $_
+ : $_;
}
sub numeric {
- $a =~ /\.(\d+)\./;
- my $a_num = $1;
- $b =~ /\.(\d+)\./;
- my $b_num = $1;
+ $a =~ /\.(\d+)\./;
+ my $a_num = $1;
+ $b =~ /\.(\d+)\./;
+ my $b_num = $1;
- $a_num <=> $b_num;
+ $a_num <=> $b_num;
}
sub spinner {
return
if ( ( time - $spinner_update_time ) < 1 );
$spinner_update_time = time;
- printf {interactive(*STDERR)} "%s\r", $spinner[ $spinner_update_time % @spinner ];
+ printf { interactive(*STDERR) } "%s\r", $spinner[ $spinner_update_time % @spinner ];
select()->flush();
}
# using current year as syslog does not record the year (nor the timezone)
# or the last year if the guessed time is in the future.
sub syslog2iso {
- my %month_map;
- @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11;
-
- my ( $month, @t ) =
- $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x;
- my $epoch =
- timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] );
- sprintf '%04d-%02d-%02d %02d:%02d:%02d',
- 1900 + (localtime)[5] - ( $epoch > time ),
- $month_map{$month} + 1, @t;
+ my %month_map;
+ @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11;
+
+ my ( $month, @t ) = $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x;
+ my $epoch =
+ timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] );
+ sprintf '%04d-%02d-%02d %02d:%02d:%02d', 1900 + (localtime)[5] - ( $epoch > time ), $month_map{$month} + 1, @t;
}
### Imported from IO::Interactive 1.022 Perl module
sub is_interactive {
- my ($out_handle) = (@_, select); # Default to default output handle
+ my ($out_handle) = ( @_, select ); # Default to default output handle
# Not interactive if output is not to terminal...
return 0 if not -t $out_handle;
# If *ARGV is opened, we're interactive if...
- if ( tied(*ARGV) or defined(fileno(ARGV)) ) { # this is what 'Scalar::Util::openhandle *ARGV' boils down to
+ if ( tied(*ARGV) or defined( fileno(ARGV) ) ) { # this is what 'Scalar::Util::openhandle *ARGV' boils down to
# ...it's currently opened to the magic '-' file
return -t *STDIN if defined $ARGV && $ARGV eq '-';
# ...it's at end-of-file and the next file is the magic '-' file
- return @ARGV>0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV;
+ return @ARGV > 0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV;
# ...it's directly attached to the terminal
return -t *ARGV;
}
### Imported from IO::Interactive 1.022 Perl module
-local (*DEV_NULL, *DEV_NULL2);
+local ( *DEV_NULL, *DEV_NULL2 );
my $dev_null;
+
BEGIN {
pipe *DEV_NULL, *DEV_NULL2
- or die "Internal error: can't create null filehandle";
+ or die "Internal error: can't create null filehandle";
$dev_null = \*DEV_NULL;
}
### Imported from IO::Interactive 1.022 Perl module
sub interactive {
- my ($out_handle) = (@_, \*STDOUT); # Default to STDOUT
+ my ($out_handle) = ( @_, \*STDOUT ); # Default to STDOUT
return &is_interactive ? $out_handle : $dev_null;
}
projects / rspamd.git / commitdiff