From 2c3a349ee9b46723f8aebfffb2e9ba4974eeec67 Mon Sep 17 00:00:00 2001 From: "cebka@lenovo-laptop" Date: Fri, 12 Mar 2010 20:36:32 +0300 Subject: [PATCH] * Add web management interface for rspamd (no design yet) * Fix Mail::Rspamd::Client --- cgi/rspamd.cgi | 479 +++++++++++++++++++++++++++++++++ perl/lib/Mail/Rspamd/Client.pm | 146 +++++++--- 2 files changed, 585 insertions(+), 40 deletions(-) create mode 100644 cgi/rspamd.cgi diff --git a/cgi/rspamd.cgi b/cgi/rspamd.cgi new file mode 100644 index 000000000..51f134d89 --- /dev/null +++ b/cgi/rspamd.cgi @@ -0,0 +1,479 @@ +#!/usr/bin/perl -w + +use strict; +use warnings; + +{ + +package RspamdWebInterface; + +use strict; +use Mail::Rspamd::Client; +use CGI qw/:standard -debug/; +use IO::Socket::INET; +use IO::String; +use Data::Dumper; + +my %cfg = ( + 'hosts' => ['localhost:11333', 'spam22'], + +); + +sub new { + my ($class, $args) = @_; + + $class = ref($class) || $class; + + my $self = { + addr => 'localhost', + port => 8080, + standalone => 0, + server_name => 'localhost', + }; + + if ($args->{'standalone'}) { + $self->{'standalone'} = 1; + } + if ($args->{'port'}) { + $self->{'port'} = $args->{'port'}; + } + if ($args->{'server_name'}) { + $self->{'server_name'} = $args->{'server_name'}; + } + if ($args->{'addr'}) { + $self->{'addr'} = $args->{'addr'}; + } + if ($args->{'config'}) { + eval($args->{'config'}); + } + + bless($self, $class); + + $self; +} + +sub _handle_ping { + my $self = shift; + my (@servers_alive, @servers_dead); + + my $rspamd = Mail::Rspamd::Client->new({}); + + # Walk throught selection of servers + foreach (@{ $cfg{'hosts'} }) { + if ($rspamd->ping($_)) { + push(@servers_alive, $_); + } + else { + push(@servers_dead, $_); + } + } + + print header; + print qq!"; + +} + +sub _show_html { + my $self = shift; + + print header, + start_html(-title=>'Rspamd control', -script=>[{-type=>'JAVASCRIPT', -src=>'http://www.google.com/jsapi'}, + {-type=>'JAVASCRIPT', -code=>'google.load("jquery", "1");'}]), + h1('Manage rspamd cluster'), + start_form(-method=>'POST', -enctype=>&CGI::MULTIPART), + "", + "
", + scrolling_list(-name => 'servers', + -multiple=>'true', + -values=>$cfg{'hosts'}, + -id=>'id_servers', + ), + "
", + button(-name=>'ping', + -value=>'Ping', + -onClick=>'$.ajax({ + url: \'/ajax\', + success: function(data) { + $(\'#servers_div\').html(data); + } + });'), + br, + "", + popup_menu (-name=>'command', + -values=>['symbols', 'check', 'stat', 'learn', 'fuzzy_add', 'fuzzy_del', 'weights', 'uptime'], + -labels=> { + 'symbols'=>'Scan symbols for message', + 'check'=>'Check if message is spam', + 'stat'=>'Check statistics', + 'learn'=>'Learn rspamd with message', + 'fuzzy_add'=>'Add fuzzy hash', + 'fuzzy_del'=>'Delete fuzzy hash', + 'weights'=>'Check weights of message', + 'uptime'=>'Get uptime', + }, + -id=>'id_command', + ), + br, + "", + textfield(-name=>'statfile', -id=>'id_statfile'), + br, + "", + filefield(-name=>'upload_file', -id=>'id_file'), + br, + "", + textarea(-name=>'message', -id=>'id_message', -rows=>10, -columns=>80), + br, + "", + textfield(-name=>'weight', -id=>'id_weight'), + br, + submit, + end_form; + + print end_html; +} + +sub _get_file { + my $self = shift; + my $fh = shift; + + my $output; + my $buffer; + + if (! $fh) { + return undef; + } + my $io_handle = $fh->handle; + + while (my $bytesread = $io_handle->read($buffer,1024)) { + $output .= $buffer; + } + + return $output; +} + + +sub _make_rfc822_message { + my $self = shift; + my $msg = shift; + + # Check whether first line is a header line + if ($msg =~ /^[^:]+:\s*\S+$/) { + # Assume that message has all headers + return $msg; + } + else { + my $output = < + +$msg +EOT + return $output; + } +} + +sub _get_message { + my $self = shift; + my $cgi = shift; + + if ($cgi->param('upload_file')) { + return $self->_get_file($cgi->upload('upload_file')); + } + elsif (my $msg = $cgi->param('message')) { + return $self->_make_rfc822_message ($msg); + } + + undef; +} + +sub _show_rspamc_result { + my $self = shift; + my $host = shift; + my $res = shift; + + if (defined($res->{error})) { + print "

Error occured: $res->{error}

"; + } + else { + while (my ($metric, $result) = each (%{ $res })) { + print "

Metric: $metric

"; + print "

Summary: $result->{isspam}, [ $result->{score} / $result->{threshold} ]

"; + print "

Symbols: "; + print join("; ", @{ $result->{symbols} }) . "

"; + print "

Urls: " . join(", ", @{ $result->{urls} }) . "

"; + foreach my $msg (@{ $result->{messages} }) { + print "

Message: $msg

"; + } + print br; + } + } +} + +sub _show_error { + my $self = shift; + my $error = shift; + + print header, + start_html(-title=>'Rspamd control', -script=>[{-type=>'JAVASCRIPT', -src=>'http://www.google.com/jsapi'}, + {-type=>'JAVASCRIPT', -code=>'google.load("jquery", "1");'}]), + h1('Results for rspamd command'), + "

Error occured: $error

", + 'Back to manage', + end_html; +} + +sub _show_control_result { + my $self = shift; + my $host = shift; + my $res = shift; + + if ($res->{error_code} == 0) { + print "

$res->{error}

"; + } + else { + print "

Error occured: $res->{error}

"; + } +} + +sub _show_results { + my $self = shift; + my $rspamd = shift; + my $res = shift; + + if (defined ($res->{error})) { + $self->_print_error($res->{error}); + return; + } + print header, + start_html(-title=>'Rspamd control', -script=>[{-type=>'JAVASCRIPT', -src=>'http://www.google.com/jsapi'}, + {-type=>'JAVASCRIPT', -code=>'google.load("jquery", "1");'}]), + h1('Results for rspamd command: ' . $rspamd->{command}); + + while (my ($host, $result) = each (%{ $res })) { + print h2('Results for host: ' . $host); + if ($rspamd->{control}) { + $self->_show_control_result ($host, $result); + } + else { + $self->_show_rspamc_result ($host, $result); + } + print hr; + } + + print 'Back to manage'; + print end_html; +} + +sub _handle_form { + my $self = shift; + my $cgi = shift; + + my @servers = $cgi->param('servers'); + if (!@servers || scalar(@servers) == 0) { + @servers = @{ $cfg{'hosts'} }; + } + my $rspamd = Mail::Rspamd::Client->new({hosts => \@servers}); + my $cmd = $cgi->param('command'); + if (!$cmd) { + return undef; + } + + my $results; + + if($cmd eq 'symbols' || $cmd eq 'check') { + my $msg = $self->_get_message($cgi); + return undef unless $msg; + $results = $rspamd->$cmd($msg); + } + elsif ($cmd eq 'learn') { + my $statfile = $cgi->param('statfile'); + return undef unless $statfile; + my $msg = $self->_get_message($cgi); + return undef unless $msg; + + $rspamd->{'statfile'} = $statfile; + if (my $weight = $cgi->param('weight')) { + $rspamd->{'weight'} = int($weight); + } + + $results = $rspamd->learn($msg); + } + elsif ($cmd eq 'fuzzy_add' || $cmd eq 'fuzzy_del') { + my $msg = $self->_get_message($cgi); + return undef unless $msg; + if (my $weight = $cgi->param('weight')) { + $rspamd->{'weight'} = int($weight); + } + + $results = $rspamd->$cmd($msg); + } + elsif ($cmd eq 'stat' || $cmd eq 'uptime') { + $results = $rspamd->$cmd(); + } + + $self->_show_results($rspamd, $results); + +} + +sub _handle_request { + my $self = shift; + my $cgi = shift; + + my $path = $cgi->path_info(); + unless ($path) { + print "CGI environment missing\n"; + return undef; + } + + print "HTTP/1.0 200 OK\r\n"; + + if ($cgi->request_method() eq 'POST') { + if (!$self->_handle_form($cgi)) { + $self->_show_error("invalid command"); + } + } + else { + if ($path =~ '^/ajax$') { + $self->_handle_ping(); + } + else { + $self->_show_html(); + } + } +} + +sub _run_standalone { + my $self = shift; + my $listen = IO::Socket::INET->new( + Listen => 5, + LocalAddr => $self->{addr}, + LocalPort => $self->{port}, + Proto => 'tcp', + ReuseAddr => 1 + ); + + unless ($listen) { + warn "unable to listen on port $self->{port}: $!\n"; + return undef; + }; + + print STDERR "waiting for connection on port $self->{port}\n"; + while (1) { + my $s = $listen->accept(); + + open(STDOUT, ">&=".fileno($s)); + open(STDIN, "<&=".fileno($s)); + + my ($req, $content); + delete $ENV{CONTENT_LENGTH}; + { + local ($/) = "\r\n"; + while () { + $req .= $_; + chomp; + last unless /\S/; + if (/^GET\s*(\S+)/) { + $ENV{REQUEST_METHOD} = 'GET'; + my ($pi, $qs) = split /\?/, $1, 2; + $ENV{'PATH_INFO'} = $pi; + $ENV{'QUERY_STRING'} = $qs; + } elsif (/^POST\s*(\S+)/) { + $ENV{REQUEST_METHOD} = 'POST'; + my ($pi, $qs) = split /\?/, $1, 2; + $ENV{'PATH_INFO'} = $pi; + $ENV{'QUERY_STRING'} = $qs; + } elsif (/^Content-Type:\s*(.*)/) { + $ENV{CONTENT_TYPE} = $1; + } elsif (/^Content-Length:\s*(.*)/) { + $ENV{CONTENT_LENGTH} = $1; + } + } + } + $ENV{SERVER_PORT} = $self->{port}; + $ENV{SERVER_NAME} = $self->{server_name}; + if (my $size = $ENV{CONTENT_LENGTH}) { + $content = ''; + while (length($content) < $size) { + my $nr = read(STDIN, $content, $size-length($content), + length($content)); + warn "read error" unless $nr; + } + } + + + close(STDIN); # n.b.: does not close socket + tie *STDIN, 'IO::String', $content; + + undef @CGI::QUERY_PARAM; + my $q = new CGI(); + $self->_handle_request($q); + + untie *STDIN; + close(STDOUT); + close($s); + } +} + +sub run { + my $self = shift; + + if ($self->{'standalone'} != 0) { + $self->_run_standalone(); + } + else { + my $q = new CGI(); + $self->_handle_request($q); + } +} + +} + +# Parse arguments + +my ($port, $standalone, $cfg, $host); + +while (my $arg = shift @ARGV) { + if ($arg =~ /^-port$/i) { + $port = shift @ARGV; + } + elsif ($arg =~ /^-standalone$/i) { + $standalone = 1; + } + elsif ($arg =~ /^-cfg$/i) { + $cfg = shift @ARGV; + } + elsif ($arg =~ /^-host$/i) { + $host = shift @ARGV; + } + else { + print STDERR <new({port=>$port, standalone=>$standalone, config=>$cfg, host=>$host})->run(); diff --git a/perl/lib/Mail/Rspamd/Client.pm b/perl/lib/Mail/Rspamd/Client.pm index d87fe16ae..580297bd3 100644 --- a/perl/lib/Mail/Rspamd/Client.pm +++ b/perl/lib/Mail/Rspamd/Client.pm @@ -200,21 +200,27 @@ sub do_all_cmd { my %res; - foreach my $hostdef (@{ $self->{'hosts'} }) { - $self->_clear_errors(); + if (!$self->{'hosts'} || scalar (@{ $self->{'hosts'} }) == 0) { + $res{'error'} = 'Hosts list is empty'; + $res{'error_code'} = 404; + } + else { + foreach my $hostdef (@{ $self->{'hosts'} }) { + $self->_clear_errors(); - my $remote = $self->_create_connection($hostdef); + my $remote = $self->_create_connection($hostdef); - if (! $remote) { - $res{$hostdef}->{error_code} = 404; - $res{$hostdef}->{error} = "Cannot connect to $hostdef"; - } - else { - if ($self->{'control'}) { - $res{$hostdef} = $self->_do_control_command ($remote, $input); + if (! $remote) { + $res{$hostdef}->{error_code} = 404; + $res{$hostdef}->{error} = "Cannot connect to $hostdef"; } else { - $res{$hostdef} = $self->_do_rspamc_command ($remote, $input); + if ($self->{'control'}) { + $res{$hostdef} = $self->_do_control_command ($remote, $input); + } + else { + $res{$hostdef} = $self->_do_rspamc_command ($remote, $input); + } } } } @@ -254,8 +260,9 @@ sub check { my ($self, $msg) = @_; $self->{command} = 'CHECK'; + $self->{control} = 0; - return $self->_do_rspamc_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 symbols @@ -288,8 +295,9 @@ sub symbols { my ($self, $msg) = @_; $self->{command} = 'SYMBOLS'; + $self->{control} = 0; - return $self->_do_rspamc_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 process @@ -322,8 +330,9 @@ sub process { my ($self, $msg) = @_; $self->{command} = 'PROCESS'; + $self->{control} = 0; - return $self->_do_rspamc_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 emails @@ -342,8 +351,9 @@ sub emails { my ($self, $msg) = @_; $self->{command} = 'EMAILS'; + $self->{control} = 0; - return $self->_do_rspamc_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 urls @@ -362,8 +372,9 @@ sub urls { my ($self, $msg) = @_; $self->{command} = 'URLS'; + $self->{control} = 0; - return $self->_do_rspamc_command ($self, $msg); + return $self->do_all_cmd ($msg); } @@ -380,8 +391,9 @@ sub learn { my ($self, $msg) = @_; $self->{command} = 'LEARN'; + $self->{control} = 1; - return $self->_do_control_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 weights @@ -395,9 +407,10 @@ This method makes a call to the spamd showing weights of message by each statfil sub weights { my ($self, $msg) = @_; - $self->{command} = 'WEIGHTS'; + $self->{command} = 'weights'; + $self->{control} = 1; - return $self->_do_control_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 fuzzy_add @@ -411,9 +424,10 @@ This method makes a call to the spamd adding specified message to fuzzy storage. sub fuzzy_add { my ($self, $msg) = @_; - $self->{command} = 'FUZZY_ADD'; + $self->{command} = 'fuzzy_add'; + $self->{control} = 1; - return $self->_do_control_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 fuzzy_del @@ -426,9 +440,10 @@ This method makes a call to the spamd removing specified message from fuzzy stor sub fuzzy_del { my ($self, $msg) = @_; - $self->{command} = 'FUZZY_DEL'; + $self->{command} = 'fuzzy_del'; + $self->{control} = 1; - return $self->_do_control_command ($self, $msg); + return $self->do_all_cmd ($msg); } =head2 stat @@ -442,9 +457,10 @@ This method makes a call to the spamd and get statistics. sub stat { my ($self) = @_; - $self->{command} = 'STAT'; + $self->{command} = 'stat'; + $self->{control} = 1; - return $self->_do_control_command ($self, undef); + return $self->do_all_cmd (undef); } =head2 uptime @@ -457,9 +473,10 @@ This method makes a call to the spamd and get uptime. sub uptime { my ($self) = @_; - $self->{command} = 'UPTIME'; + $self->{command} = 'uptime'; + $self->{control} = 1; - return $self->_do_control_command ($self, undef); + return $self->do_all_cmd (undef); } =head2 counters @@ -472,9 +489,10 @@ This method makes a call to the spamd and get counters. sub counters { my ($self) = @_; - $self->{command} = 'UPTIME'; + $self->{command} = 'counters'; + $self->{control} = 1; - return $self->_do_control_command ($self, undef); + return $self->do_all_cmd (undef); } =head2 ping @@ -488,15 +506,25 @@ if the server responded correctly. =cut sub ping { - my ($self) = @_; - - my $remote = $self->_create_connection(); - - return 0 unless ($remote); + my $self = shift; + my $host = shift; + + my $remote; + $self->{control} = 0; + if (defined($host)) { + $remote = $self->_create_connection($host); + } + else { + # Create connection to random host from cluster + $remote = $self->_create_connection(); + } + + return undef unless $remote; local $SIG{PIPE} = 'IGNORE'; if (!(syswrite($remote, "PING $PROTOVERSION$EOL"))) { $self->_mark_dead($remote); + close($remote); return 0; } syswrite($remote, $EOL); @@ -679,7 +707,9 @@ This method returns one server from rspamd cluster or undef if there are no suit =cut sub _select_server { my($self) = @_; - + + return undef unless $self->{alive_hosts}; + $self->_revive_dead(); my $alive_num = scalar(@{$self->{alive_hosts}}); if (!$alive_num) { @@ -712,6 +742,7 @@ This method marks upstream as dead for some time. It can be revived by _revive_d sub _mark_dead { my ($self, $server) = @_; + return undef unless $self->{hosts}; my $now = time(); $self->{dead_hosts}->{$server} = { host => $server, @@ -791,7 +822,7 @@ sub _do_rspamc_command { my ($self, $remote, $msg) = @_; my %metrics; - + my ($in, $res); my $msgsize = length($msg.$EOL); @@ -799,7 +830,12 @@ sub _do_rspamc_command { if (!(syswrite($remote, "$self->{command} $PROTOVERSION$EOL"))) { $self->_mark_dead($remote); - return 0; + my %r = ( + error => 'cannot connect to rspamd', + error_code => 502, + ); + close($remote); + return \%r; } syswrite $remote, "Content-length: $msgsize$EOL"; syswrite $remote, "User: $self->{username}$EOL" if ($self->{username}); @@ -815,9 +851,15 @@ sub _do_rspamc_command { syswrite $remote, $msg; syswrite $remote, $EOL; - return undef unless $self->_get_io_readiness($remote, 0); + unless ($self->_get_io_readiness($remote, 0)) { + close $remote; + my %r = ( + error => 'timed out while waiting for reply', + error_code => 502, + ); + return \%r; + } - my ($in, $res); my $offset = 0; do { $res = sysread($remote, $in, 512, $offset); @@ -832,7 +874,14 @@ sub _do_rspamc_command { $self->{resp_code} = $resp_code; $self->{resp_msg} = $resp_msg; - return undef unless (defined($resp_code) && $resp_code == 0); + unless (defined($resp_code) && $resp_code == 0) { + close $remote; + my %r = ( + error => 'invalid reply', + error_code => 500, + ); + return \%r + } my $cur_metric; my @lines = split (/^/, $in); @@ -888,6 +937,7 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } @@ -896,6 +946,7 @@ sub _do_control_command { if ($greeting !~ /^Rspamd version/) { $res{error} = "Not rspamd greeting line $greeting"; $res{error_code} = 500; + close($remote); return \%res; } } @@ -904,6 +955,7 @@ sub _do_control_command { if (!$self->{'statfile'}) { $res{error} = "Statfile is not specified to learn command"; $res{error_code} = 500; + close($remote); return \%res; } @@ -914,16 +966,19 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } if (defined (my $reply = <$remote>)) { if ($reply =~ /^learn ok, sum weight: ([0-9.]+)/) { $res{error} = "Learn succeed. Sum weight: $1\n"; + close($remote); return \%res; } else { $res{error_code} = 500; $res{error} = "Learn failed\n"; + close($remote); return \%res; } } @@ -931,6 +986,7 @@ sub _do_control_command { else { $res{error_code} = 403; $res{error} = "Authentication failed\n"; + close($remote); return \%res; } } @@ -938,6 +994,7 @@ sub _do_control_command { if (!$self->{'statfile'}) { $res{error_code} = 500; $res{error} = "Statfile is not specified to weights command"; + close($remote); return \%res; } @@ -948,6 +1005,7 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } while (defined (my $reply = <$remote>)) { @@ -961,6 +1019,7 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } while (defined (my $line = <$remote>)) { @@ -971,6 +1030,7 @@ sub _do_control_command { else { $res{error_code} = 403; $res{error} = "Authentication failed\n"; + close($remote); return \%res; } } @@ -982,16 +1042,19 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } if (defined (my $reply = <$remote>)) { if ($reply =~ /^OK/) { $res{error} = $self->{'command'} . " succeed\n"; + close($remote); return \%res; } else { $res{error_code} = 500; $res{error} = $self->{'command'} . " failed\n"; + close($remote); return \%res; } } @@ -999,6 +1062,7 @@ sub _do_control_command { else { $res{error_code} = 403; $res{error} = "Authentication failed\n"; + close($remote); return \%res; } @@ -1008,6 +1072,7 @@ sub _do_control_command { unless ($self->_get_io_readiness($remote, 0)) { $res{error} = "Timeout while reading data from socket"; $res{error_code} = 501; + close($remote); return \%res; } while (defined (my $line = <$remote>)) { @@ -1016,6 +1081,7 @@ sub _do_control_command { } } + close($remote); return \%res; } -- 2.39.5