mirrors
/
rspamd
peilaus alkaen https://github.com/vstakhov/rspamd.git
Tarkkaile 1
Äänestä 0
Fork 0
Koodi Ongelmat 0 Julkaisut 159 Wiki Activity
Selaa lähdekoodia

[Fix] Various fixes to asn module 

Contributed by: @citrin
tags/2.0
Vsevolod Stakhov 4 vuotta sitten
vanhempi
7eaa2fde30
commit
e7b7f84e3e
1 muutettua tiedostoa jossa 186 lisäystä ja 230 poistoa
Jaettu näkymä Näytä diff tilastot
  1. 186
    230
      utils/asn.pl

+ 186
- 230
utils/asn.pl Näytä tiedosto

@@ -1,29 +1,31 @@
#!/usr/bin/env perl
#
# Ansible managed
#

use warnings;
use strict;
use autodie;

use File::Basename;
use File::Fetch;
use Getopt::Long;
use IPC::Cmd qw/can_run/;
use Pod::Usage;

use LWP::Simple;
use PerlIO::gzip;
use URI;
use FindBin;
use lib "$FindBin::Bin/extlib/lib/perl5";

$LWP::Simple::ua->show_progress(1);
use URI;

my %config = (
asn_sources => [
'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
],
bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
asn_sources => [
'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
],
bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
);

my $download_asn = 0;
@@ -39,278 +41,230 @@ my $v6_zone = "asn6.rspamd.com";
my $v4_file = "asn.zone";
my $v6_file = "asn6.zone";
my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
my $use_bgpdump = 0;

GetOptions(
"download-asn" => \$download_asn,
"download-bgp" => \$download_bgp,
"4!" => \$v4,
"6!" => \$v6,
"parse!" => \$parse,
"target=s" => \$download_target,
"zone-v4=s" => \$v4_zone,
"zone-v6=s" => \$v6_zone,
"file-v4=s" => \$v4_file,
"file-v6=s" => \$v6_file,
"ns-server=s@" => \$ns_servers,
"help|?" => \$help,
"man" => \$man,
) or pod2usage(2);
"download-asn" => \$download_asn,
"download-bgp" => \$download_bgp,
"4!" => \$v4,
"6!" => \$v6,
"parse!" => \$parse,
"target=s" => \$download_target,
"zone-v4=s" => \$v4_zone,
"zone-v6=s" => \$v6_zone,
"file-v4=s" => \$v4_file,
"file-v6=s" => \$v6_file,
"ns-server=s@" => \$ns_servers,
"help|?" => \$help,
"man" => \$man
) or
pod2usage(2);

pod2usage(1) if $help;
pod2usage( -exitval => 0, -verbose => 2 ) if $man;

my $bgpdump_path = can_run('bgpdump')
or warn 'bgpdump is not found, will try to use Net::MRT instead; results can be incomplete';

sub download_file {
my ($u) = @_;

print "Fetching $u\n";
my $ff = File::Fetch->new( uri => $u );
my $where = $ff->fetch( to => $download_target ) or die $ff->error;

return $where;
}
pod2usage(-exitval => 0, -verbose => 2) if $man;

if ($download_asn) {
foreach my $u ( @{ $config{'asn_sources'} } ) {
foreach my $u (@{ $config{'asn_sources'} }) {
download_file($u);
}
}

if ($download_bgp) {
foreach my $u ( @{ $config{'bgp_sources'} } ) {
foreach my $u (@{ $config{'bgp_sources'} }) {
download_file($u);
}
}

if ( !$parse ) {
if (!$parse) {
exit 0;
}

my $v4_fh;
my $v6_fh;

if ($v4) {
open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!";
print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
foreach my $ns ( @{$ns_servers} ) {
print $v4_fh "\$NS 43200 $ns\n";
}
}
if ($v6) {
open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!";
print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
foreach my $ns ( @{$ns_servers} ) {
print $v6_fh "\$NS 43200 $ns\n";
}
}

sub is_bougus_asn {
my $as = shift;

# 64496-64511 Reserved for use in documentation and sample code
# 64512-65534 Designated for private use
# 65535 Reserved
# 65536-65551 Reserved for use in documentation and sample code
# 65552-131071 Reserved
return 1 if $as >= 64496 && $as <= 131071;

# 4294967295
return 1 if $as == 4294967295;
# Prefix to ASN map
my $networks = { 4 => {}, 6 => {} };

# AS0 is reserved
# AS1 is legal AS, but in most cases used by others without permission
# of owner (probably lame admins use AS1 as private AS).
return 1 if $as <= 1;
foreach my $u (@{ $config{'bgp_sources'} }) {
my $parsed = URI->new($u);
my $fname = $download_target . '/' . basename($parsed->path);

return 0;
}
use constant {
F_MARKER => 0,
F_TIMESTAMP => 1,
F_PEER_IP => 3,
F_PEER_AS => 4,
F_PREFIX => 5,
F_AS_PATH => 6,
F_ORIGIN => 7,
};

# Now load BGP data
my $networks = {};
open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";

foreach my $u ( @{ $config{'bgp_sources'} } ) {
my $parsed = URI->new($u);
my $fname = $download_target . '/' . basename( $parsed->path );

if ($bgpdump_path) {
use constant {
F_MARKER => 0,
F_TIMESTAMP => 1,
F_PEER_IP => 3,
F_PEER_AS => 4,
F_PREFIX => 5,
F_AS_PATH => 6,
F_ORIGIN => 7,
};

open( my $bgpd, '-|', "$bgpdump_path -v -M $fname" ) or die "can't start bgpdump: $!";

while (<$bgpd>) {
chomp;
my @e = split /\|/;
if ( $e[F_MARKER] ne 'TABLE_DUMP2' ) {
warn "bad line: $_\n";
next;
}
while (<$bgpd>) {
chomp;
my @e = split /\|/;
if ($e[F_MARKER] ne 'TABLE_DUMP2') {
warn "bad line: $_\n";
next;
}

my $origin_as;
my $prefix = $e[F_PREFIX];
my $ipv6 = 0;
my $origin_as;
my $prefix = $e[F_PREFIX];
my $ip_ver = 6;

if ( $prefix !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/ ) {
$ipv6 = 1;
}
if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
$ip_ver = 4;
}

if ( $e[F_AS_PATH] ) {
if ($e[F_AS_PATH]) {

# not empty AS_PATH
my @as_path = split /\s/, $e[F_AS_PATH];
$origin_as = pop @as_path;
# not empty AS_PATH
my @as_path = split /\s/, $e[F_AS_PATH];
$origin_as = pop @as_path;

if ( substr( $origin_as, 0, 1 ) eq '{' ) {
if (substr($origin_as, 0, 1) eq '{') {

# route is aggregated
if ( $origin_as =~ /^{(\d+)}$/ ) {
# route is aggregated
if ($origin_as =~ /^{(\d+)}$/) {

# single AS aggregated, just remove { } around
$origin_as = $1;
}
else {
# use previous AS from AS_PATH
$origin_as = pop @as_path;
}
}
# single AS aggregated, just remove { } around
$origin_as = $1;
} else {

# strip bogus AS
while ( is_bougus_asn($origin_as) ) {
# use previous AS from AS_PATH
$origin_as = pop @as_path;
last if scalar @as_path == 0;
}
}

# empty AS_PATH or all AS_PATH elements is stripped as bogus - use PEER_AS is origin AS
$origin_as //= $e[F_PEER_AS];

if ( !$networks->{$origin_as} ) {
if ( !$ipv6 ) {
$networks->{$origin_as} = { nets_v4 => [$prefix], nets_v6 => [] };
}
else {
$networks->{$origin_as} = { nets_v6 => [$prefix], nets_v4 => [] };
}
}
else {
if ( !$ipv6 ) {
push @{ $networks->{$origin_as}->{'nets_v4'} }, $prefix;
}
else {
push @{ $networks->{$origin_as}->{'nets_v6'} }, $prefix;
}
}
}
}
else {
require Net::MRT;
$Net::MRT::USE_RFC4760 = -1;

open( my $fh, "<:gzip", $fname )
or die "Cannot open $fname: $!";
while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) {
if ( $dd->{'prefix'} && $dd->{'bits'} ) {
next if $dd->{'subtype'} == 2 and !$v4;
next if $dd->{'subtype'} == 4 and !$v6;
my $entry = $dd->{'entries'}->[0];
my $net = $dd->{'prefix'} . '/' . $dd->{'bits'};
if ( $entry && $entry->{'AS_PATH'} ) {
my $as = $entry->{'AS_PATH'}->[-1];
if ( ref($as) eq "ARRAY" ) {
$as = @{$as}[0];
}

next if ( is_bougus_asn($as) );

if ( !$networks->{$as} ) {
if ( $dd->{'subtype'} == 2 ) {
$networks->{$as} = { nets_v4 => [$net], nets_v6 => [] };
}
else {
$networks->{$as} = { nets_v6 => [$net], nets_v4 => [] };
}
}
else {

if ( $dd->{'subtype'} == 2 ) {
push @{ $networks->{$as}->{'nets_v4'} }, $net;
}
else {
push @{ $networks->{$as}->{'nets_v6'} }, $net;
}
}
}
# strip bogus AS
while (is_bougus_asn($origin_as)) {
$origin_as = pop @as_path;
last if scalar @as_path == 0;
}
}

# empty AS_PATH or all AS_PATH elements was stripped as bogus - use
# PEER_AS as origin AS
$origin_as //= $e[F_PEER_AS];

$networks->{$ip_ver}{$prefix} = int($origin_as);
}
}

# Remove default routes
delete $networks->{4}{'0.0.0.0/0'};
delete $networks->{6}{'::/0'};

# Now roughly detect countries
foreach my $u ( @{ $config{'asn_sources'} } ) {
my $as_info = {};

# RIR statistics exchange format
# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
# first 7 fields for this two formats are same
use constant {
F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc}
F_CC => 1, # ISO 3166 2-letter contry code
F_TYPE => 2, # {asn,ipv4,ipv6}
F_START => 3,
F_VALUE => 4,
F_DATE => 5,
F_STATUS => 6,
};

foreach my $u (@{ $config{'asn_sources'} }) {
my $parsed = URI->new($u);
my $fname = $download_target . '/' . basename( $parsed->path );
open( my $fh, "<", $fname ) or die "Cannot open $fname: $!";
my $fname = $download_target . '/' . basename($parsed->path);
open(my $fh, "<", $fname) or die "Cannot open $fname: $!";

while (<$fh>) {
next if /^\#/;
chomp;
my @elts = split /\|/;

if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) {
my $as_start = int( $elts[3] );
my $as_end = $as_start + int( $elts[4] );

for ( my $as = $as_start ; $as < $as_end ; $as++ ) {
my $real_as = $as;
if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
my $as_start = int($elts[F_START]);
my $as_end = $as_start + int($elts[F_VALUE]) - 1;

if ( ref($as) eq "ARRAY" ) {
$real_as = @{$as}[0];
}

if ( $networks->{"$real_as"} ) {
$networks->{"$real_as"}->{'country'} = $elts[1];
$networks->{"$real_as"}->{'rir'} = $elts[0];
}
for my $as ($as_start .. $as_end) {
$as_info->{$as}{'country'} = $elts[F_CC];
$as_info->{$as}{'rir'} = $elts[F_REGISTRY];
}
}
}
}

while ( my ( $k, $v ) = each( %{$networks} ) ) {
if ($v4) {
foreach my $n ( @{ $v->{'nets_v4'} } ) {
# Write zone files
my $ns_list = join ' ', @{$ns_servers};
my $zone_header = << "EOH";
\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
\$NS 43200 $ns_list
EOH

# "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
if ( $v->{'country'} ) {
printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
}
else {
printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
}
}
if ($v4) {
open my $v4_fh, '>', ".$v4_file.tmp";
print $v4_fh $zone_header;

while (my ($net, $asn) = each %{ $networks->{4} }) {
my $country = $as_info->{$asn}{'country'} || 'UN';
my $rir = $as_info->{$asn}{'rir'} || 'UN';

# "15169|8.8.8.0/24|US|arin|" for 8.8.8.8
printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
}
if ($v6) {
foreach my $n ( @{ $v->{'nets_v6'} } ) {

# "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
if ( $v->{'country'} ) {
printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
}
else {
printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
}
}
close $v4_fh;
rename ".$v4_file.tmp", $v4_file;
}

if ($v6) {
open my $v6_fh, '>', ".$v6_file.tmp";
print $v6_fh $zone_header;

while (my ($net, $asn) = each %{ $networks->{6} }) {
my $country = $as_info->{$asn}{'country'} || 'UN';
my $rir = $as_info->{$asn}{'rir'} || 'UN';

# "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
}

close $v6_fh;
rename ".$v6_file.tmp", $v6_file;
}

exit 0;

########################################################################

sub download_file {
my ($url) = @_;

local $File::Fetch::WARN = 0;
local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad

my $ff = File::Fetch->new(uri => $url);
my $where = $ff->fetch(to => $download_target) or
die "$url: ", $ff->error;

return $where;
}

# Returns true if AS number is bogus
# e. g. a private AS.
# List of allocated and reserved AS:
# https://www.iana.org/assignments/as-numbers/as-numbers.txt
sub is_bougus_asn {
my $as = shift;

# 64496-64511 Reserved for use in documentation and sample code
# 64512-65534 Designated for private use
# 65535 Reserved
# 65536-65551 Reserved for use in documentation and sample code
# 65552-131071 Reserved
return 1 if $as >= 64496 && $as <= 131071;

# Reserved (RFC6996, RFC7300, RFC7607)
return 1 if $as == 0 || $as >= 4200000000;

return 0;
}

__END__
@@ -324,8 +278,8 @@ asn.pl - download and parse ASN data for Rspamd
asn.pl [options]

Options:
--download-asn Download ASN data from RIR
--download-bgp Download GeoIP data from Maxmind
--download-asn Download ASN data from RIRs
--download-bgp Download BGP full view dump from RIPE RIS
--target Where to download files (default: current dir)
--zone-v4 IPv4 zone (default: asn.rspamd.com)
--zone-v6 IPv6 zone (default: asn6.rspamd.com)
@@ -365,3 +319,5 @@ Prints the manual page and exits.
B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.

=cut

# vim: et:ts=4:sw=4

Write Preview
Loading…
Peruuta
Tallenna