From: Vsevolod Stakhov Date: Mon, 22 Jul 2019 12:07:21 +0000 (+0100) Subject: [Fix] Various fixes to asn module X-Git-Tag: 2.0~520 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=e7b7f84e3e31a252c87a75c1719555de039e9e72;p=rspamd.git [Fix] Various fixes to asn module Contributed by: @citrin --- diff --git a/utils/asn.pl b/utils/asn.pl index 8cc6e4afd..15d7d7862 100644 --- a/utils/asn.pl +++ b/utils/asn.pl @@ -1,29 +1,31 @@ #!/usr/bin/env perl +# +# Ansible managed +# use warnings; use strict; +use autodie; use File::Basename; use File::Fetch; use Getopt::Long; -use IPC::Cmd qw/can_run/; use Pod::Usage; -use LWP::Simple; -use PerlIO::gzip; -use URI; +use FindBin; +use lib "$FindBin::Bin/extlib/lib/perl5"; -$LWP::Simple::ua->show_progress(1); +use URI; my %config = ( - asn_sources => [ - 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', - 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', - 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', - 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', - 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest' - ], - bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz'] + asn_sources => [ + 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', + 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', + 'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', + 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', + 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest' + ], + bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz'] ); my $download_asn = 0; @@ -39,278 +41,230 @@ my $v6_zone = "asn6.rspamd.com"; my $v4_file = "asn.zone"; my $v6_file = "asn6.zone"; my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ]; -my $use_bgpdump = 0; GetOptions( - "download-asn" => \$download_asn, - "download-bgp" => \$download_bgp, - "4!" => \$v4, - "6!" => \$v6, - "parse!" => \$parse, - "target=s" => \$download_target, - "zone-v4=s" => \$v4_zone, - "zone-v6=s" => \$v6_zone, - "file-v4=s" => \$v4_file, - "file-v6=s" => \$v6_file, - "ns-server=s@" => \$ns_servers, - "help|?" => \$help, - "man" => \$man, -) or pod2usage(2); + "download-asn" => \$download_asn, + "download-bgp" => \$download_bgp, + "4!" => \$v4, + "6!" => \$v6, + "parse!" => \$parse, + "target=s" => \$download_target, + "zone-v4=s" => \$v4_zone, + "zone-v6=s" => \$v6_zone, + "file-v4=s" => \$v4_file, + "file-v6=s" => \$v6_file, + "ns-server=s@" => \$ns_servers, + "help|?" => \$help, + "man" => \$man +) or + pod2usage(2); pod2usage(1) if $help; -pod2usage( -exitval => 0, -verbose => 2 ) if $man; - -my $bgpdump_path = can_run('bgpdump') - or warn 'bgpdump is not found, will try to use Net::MRT instead; results can be incomplete'; - -sub download_file { - my ($u) = @_; - - print "Fetching $u\n"; - my $ff = File::Fetch->new( uri => $u ); - my $where = $ff->fetch( to => $download_target ) or die $ff->error; - - return $where; -} +pod2usage(-exitval => 0, -verbose => 2) if $man; if ($download_asn) { - foreach my $u ( @{ $config{'asn_sources'} } ) { + foreach my $u (@{ $config{'asn_sources'} }) { download_file($u); } } if ($download_bgp) { - foreach my $u ( @{ $config{'bgp_sources'} } ) { + foreach my $u (@{ $config{'bgp_sources'} }) { download_file($u); } } -if ( !$parse ) { +if (!$parse) { exit 0; } -my $v4_fh; -my $v6_fh; - -if ($v4) { - open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!"; - print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; - foreach my $ns ( @{$ns_servers} ) { - print $v4_fh "\$NS 43200 $ns\n"; - } -} -if ($v6) { - open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!"; - print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n"; - foreach my $ns ( @{$ns_servers} ) { - print $v6_fh "\$NS 43200 $ns\n"; - } -} - -sub is_bougus_asn { - my $as = shift; - - # 64496-64511 Reserved for use in documentation and sample code - # 64512-65534 Designated for private use - # 65535 Reserved - # 65536-65551 Reserved for use in documentation and sample code - # 65552-131071 Reserved - return 1 if $as >= 64496 && $as <= 131071; - - # 4294967295 - return 1 if $as == 4294967295; +# Prefix to ASN map +my $networks = { 4 => {}, 6 => {} }; - # AS0 is reserved - # AS1 is legal AS, but in most cases used by others without permission - # of owner (probably lame admins use AS1 as private AS). - return 1 if $as <= 1; +foreach my $u (@{ $config{'bgp_sources'} }) { + my $parsed = URI->new($u); + my $fname = $download_target . '/' . basename($parsed->path); - return 0; -} + use constant { + F_MARKER => 0, + F_TIMESTAMP => 1, + F_PEER_IP => 3, + F_PEER_AS => 4, + F_PREFIX => 5, + F_AS_PATH => 6, + F_ORIGIN => 7, + }; -# Now load BGP data -my $networks = {}; + open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!"; -foreach my $u ( @{ $config{'bgp_sources'} } ) { - my $parsed = URI->new($u); - my $fname = $download_target . '/' . basename( $parsed->path ); - - if ($bgpdump_path) { - use constant { - F_MARKER => 0, - F_TIMESTAMP => 1, - F_PEER_IP => 3, - F_PEER_AS => 4, - F_PREFIX => 5, - F_AS_PATH => 6, - F_ORIGIN => 7, - }; - - open( my $bgpd, '-|', "$bgpdump_path -v -M $fname" ) or die "can't start bgpdump: $!"; - - while (<$bgpd>) { - chomp; - my @e = split /\|/; - if ( $e[F_MARKER] ne 'TABLE_DUMP2' ) { - warn "bad line: $_\n"; - next; - } + while (<$bgpd>) { + chomp; + my @e = split /\|/; + if ($e[F_MARKER] ne 'TABLE_DUMP2') { + warn "bad line: $_\n"; + next; + } - my $origin_as; - my $prefix = $e[F_PREFIX]; - my $ipv6 = 0; + my $origin_as; + my $prefix = $e[F_PREFIX]; + my $ip_ver = 6; - if ( $prefix !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/ ) { - $ipv6 = 1; - } + if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) { + $ip_ver = 4; + } - if ( $e[F_AS_PATH] ) { + if ($e[F_AS_PATH]) { - # not empty AS_PATH - my @as_path = split /\s/, $e[F_AS_PATH]; - $origin_as = pop @as_path; + # not empty AS_PATH + my @as_path = split /\s/, $e[F_AS_PATH]; + $origin_as = pop @as_path; - if ( substr( $origin_as, 0, 1 ) eq '{' ) { + if (substr($origin_as, 0, 1) eq '{') { - # route is aggregated - if ( $origin_as =~ /^{(\d+)}$/ ) { + # route is aggregated + if ($origin_as =~ /^{(\d+)}$/) { - # single AS aggregated, just remove { } around - $origin_as = $1; - } - else { - # use previous AS from AS_PATH - $origin_as = pop @as_path; - } - } + # single AS aggregated, just remove { } around + $origin_as = $1; + } else { - # strip bogus AS - while ( is_bougus_asn($origin_as) ) { + # use previous AS from AS_PATH $origin_as = pop @as_path; - last if scalar @as_path == 0; } } - # empty AS_PATH or all AS_PATH elements is stripped as bogus - use PEER_AS is origin AS - $origin_as //= $e[F_PEER_AS]; - - if ( !$networks->{$origin_as} ) { - if ( !$ipv6 ) { - $networks->{$origin_as} = { nets_v4 => [$prefix], nets_v6 => [] }; - } - else { - $networks->{$origin_as} = { nets_v6 => [$prefix], nets_v4 => [] }; - } - } - else { - if ( !$ipv6 ) { - push @{ $networks->{$origin_as}->{'nets_v4'} }, $prefix; - } - else { - push @{ $networks->{$origin_as}->{'nets_v6'} }, $prefix; - } - } - } - } - else { - require Net::MRT; - $Net::MRT::USE_RFC4760 = -1; - - open( my $fh, "<:gzip", $fname ) - or die "Cannot open $fname: $!"; - while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) { - if ( $dd->{'prefix'} && $dd->{'bits'} ) { - next if $dd->{'subtype'} == 2 and !$v4; - next if $dd->{'subtype'} == 4 and !$v6; - my $entry = $dd->{'entries'}->[0]; - my $net = $dd->{'prefix'} . '/' . $dd->{'bits'}; - if ( $entry && $entry->{'AS_PATH'} ) { - my $as = $entry->{'AS_PATH'}->[-1]; - if ( ref($as) eq "ARRAY" ) { - $as = @{$as}[0]; - } - - next if ( is_bougus_asn($as) ); - - if ( !$networks->{$as} ) { - if ( $dd->{'subtype'} == 2 ) { - $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] }; - } - else { - $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] }; - } - } - else { - - if ( $dd->{'subtype'} == 2 ) { - push @{ $networks->{$as}->{'nets_v4'} }, $net; - } - else { - push @{ $networks->{$as}->{'nets_v6'} }, $net; - } - } - } + # strip bogus AS + while (is_bougus_asn($origin_as)) { + $origin_as = pop @as_path; + last if scalar @as_path == 0; } } + + # empty AS_PATH or all AS_PATH elements was stripped as bogus - use + # PEER_AS as origin AS + $origin_as //= $e[F_PEER_AS]; + + $networks->{$ip_ver}{$prefix} = int($origin_as); } } +# Remove default routes +delete $networks->{4}{'0.0.0.0/0'}; +delete $networks->{6}{'::/0'}; + # Now roughly detect countries -foreach my $u ( @{ $config{'asn_sources'} } ) { +my $as_info = {}; + +# RIR statistics exchange format +# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format +# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf +# first 7 fields for this two formats are same +use constant { + F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc} + F_CC => 1, # ISO 3166 2-letter contry code + F_TYPE => 2, # {asn,ipv4,ipv6} + F_START => 3, + F_VALUE => 4, + F_DATE => 5, + F_STATUS => 6, +}; + +foreach my $u (@{ $config{'asn_sources'} }) { my $parsed = URI->new($u); - my $fname = $download_target . '/' . basename( $parsed->path ); - open( my $fh, "<", $fname ) or die "Cannot open $fname: $!"; + my $fname = $download_target . '/' . basename($parsed->path); + open(my $fh, "<", $fname) or die "Cannot open $fname: $!"; while (<$fh>) { next if /^\#/; chomp; my @elts = split /\|/; - if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) { - my $as_start = int( $elts[3] ); - my $as_end = $as_start + int( $elts[4] ); - - for ( my $as = $as_start ; $as < $as_end ; $as++ ) { - my $real_as = $as; + if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') { + my $as_start = int($elts[F_START]); + my $as_end = $as_start + int($elts[F_VALUE]) - 1; - if ( ref($as) eq "ARRAY" ) { - $real_as = @{$as}[0]; - } - - if ( $networks->{"$real_as"} ) { - $networks->{"$real_as"}->{'country'} = $elts[1]; - $networks->{"$real_as"}->{'rir'} = $elts[0]; - } + for my $as ($as_start .. $as_end) { + $as_info->{$as}{'country'} = $elts[F_CC]; + $as_info->{$as}{'rir'} = $elts[F_REGISTRY]; } } } } -while ( my ( $k, $v ) = each( %{$networks} ) ) { - if ($v4) { - foreach my $n ( @{ $v->{'nets_v4'} } ) { +# Write zone files +my $ns_list = join ' ', @{$ns_servers}; +my $zone_header = << "EOH"; +\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300 +\$NS 43200 $ns_list +EOH - # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 - if ( $v->{'country'} ) { - printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; - } - else { - printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; - } - } +if ($v4) { + open my $v4_fh, '>', ".$v4_file.tmp"; + print $v4_fh $zone_header; + + while (my ($net, $asn) = each %{ $networks->{4} }) { + my $country = $as_info->{$asn}{'country'} || 'UN'; + my $rir = $as_info->{$asn}{'rir'} || 'UN'; + + # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8 + printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir; } - if ($v6) { - foreach my $n ( @{ $v->{'nets_v6'} } ) { - # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8 - if ( $v->{'country'} ) { - printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'}; - } - else { - printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN'; - } - } + close $v4_fh; + rename ".$v4_file.tmp", $v4_file; +} + +if ($v6) { + open my $v6_fh, '>', ".$v6_file.tmp"; + print $v6_fh $zone_header; + + while (my ($net, $asn) = each %{ $networks->{6} }) { + my $country = $as_info->{$asn}{'country'} || 'UN'; + my $rir = $as_info->{$asn}{'rir'} || 'UN'; + + # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111 + printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir; } + + close $v6_fh; + rename ".$v6_file.tmp", $v6_file; +} + +exit 0; + +######################################################################## + +sub download_file { + my ($url) = @_; + + local $File::Fetch::WARN = 0; + local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad + + my $ff = File::Fetch->new(uri => $url); + my $where = $ff->fetch(to => $download_target) or + die "$url: ", $ff->error; + + return $where; +} + +# Returns true if AS number is bogus +# e. g. a private AS. +# List of allocated and reserved AS: +# https://www.iana.org/assignments/as-numbers/as-numbers.txt +sub is_bougus_asn { + my $as = shift; + + # 64496-64511 Reserved for use in documentation and sample code + # 64512-65534 Designated for private use + # 65535 Reserved + # 65536-65551 Reserved for use in documentation and sample code + # 65552-131071 Reserved + return 1 if $as >= 64496 && $as <= 131071; + + # Reserved (RFC6996, RFC7300, RFC7607) + return 1 if $as == 0 || $as >= 4200000000; + + return 0; } __END__ @@ -324,8 +278,8 @@ asn.pl - download and parse ASN data for Rspamd asn.pl [options] Options: - --download-asn Download ASN data from RIR - --download-bgp Download GeoIP data from Maxmind + --download-asn Download ASN data from RIRs + --download-bgp Download BGP full view dump from RIPE RIS --target Where to download files (default: current dir) --zone-v4 IPv4 zone (default: asn.rspamd.com) --zone-v6 IPv6 zone (default: asn6.rspamd.com) @@ -365,3 +319,5 @@ Prints the manual page and exits. B is intended to download ASN data and GeoIP data and create a rbldnsd zone. =cut + +# vim: et:ts=4:sw=4 \ No newline at end of file