[Fix] Various fixes to asn module

Contributed by: @citrin

1 files changed, 186 insertions, 230 deletions

diff --git a/utils/asn.pl b/utils/asn.pl
index 8cc6e4afd..15d7d7862 100644
--- a/utils/asn.pl
+++ b/utils/asn.pl
@@ -1,29 +1,31 @@
 #!/usr/bin/env perl
+#
+# Ansible managed
+#
 
 use warnings;
 use strict;
+use autodie;
 
 use File::Basename;
 use File::Fetch;
 use Getopt::Long;
-use IPC::Cmd qw/can_run/;
 use Pod::Usage;
 
-use LWP::Simple;
-use PerlIO::gzip;
-use URI;
+use FindBin;
+use lib "$FindBin::Bin/extlib/lib/perl5";
 
-$LWP::Simple::ua->show_progress(1);
+use URI;
 
 my %config = (
-    asn_sources => [
-        'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
-        'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
-        'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
-        'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
-        'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
-    ],
-    bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
+  asn_sources => [
+    'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
+    'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
+    'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
+    'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
+    'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
+  ],
+  bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
 );
 
 my $download_asn    = 0;
@@ -39,278 +41,230 @@ my $v6_zone         = "asn6.rspamd.com";
 my $v4_file         = "asn.zone";
 my $v6_file         = "asn6.zone";
 my $ns_servers      = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
-my $use_bgpdump     = 0;
 
 GetOptions(
-    "download-asn" => \$download_asn,
-    "download-bgp" => \$download_bgp,
-    "4!"           => \$v4,
-    "6!"           => \$v6,
-    "parse!"       => \$parse,
-    "target=s"     => \$download_target,
-    "zone-v4=s"    => \$v4_zone,
-    "zone-v6=s"    => \$v6_zone,
-    "file-v4=s"    => \$v4_file,
-    "file-v6=s"    => \$v6_file,
-    "ns-server=s@" => \$ns_servers,
-    "help|?"       => \$help,
-    "man"          => \$man,
-) or pod2usage(2);
+  "download-asn" => \$download_asn,
+  "download-bgp" => \$download_bgp,
+  "4!"           => \$v4,
+  "6!"           => \$v6,
+  "parse!"       => \$parse,
+  "target=s"     => \$download_target,
+  "zone-v4=s"    => \$v4_zone,
+  "zone-v6=s"    => \$v6_zone,
+  "file-v4=s"    => \$v4_file,
+  "file-v6=s"    => \$v6_file,
+  "ns-server=s@" => \$ns_servers,
+  "help|?"       => \$help,
+  "man"          => \$man
+) or
+  pod2usage(2);
 
 pod2usage(1) if $help;
-pod2usage( -exitval => 0, -verbose => 2 ) if $man;
-
-my $bgpdump_path = can_run('bgpdump')
-  or warn 'bgpdump is not found, will try to use Net::MRT instead; results can be incomplete';
-
-sub download_file {
-    my ($u) = @_;
-
-    print "Fetching $u\n";
-    my $ff    = File::Fetch->new( uri => $u );
-    my $where = $ff->fetch( to => $download_target ) or die $ff->error;
-
-    return $where;
-}
+pod2usage(-exitval => 0, -verbose => 2) if $man;
 
 if ($download_asn) {
-    foreach my $u ( @{ $config{'asn_sources'} } ) {
+    foreach my $u (@{ $config{'asn_sources'} }) {
         download_file($u);
     }
 }
 
 if ($download_bgp) {
-    foreach my $u ( @{ $config{'bgp_sources'} } ) {
+    foreach my $u (@{ $config{'bgp_sources'} }) {
         download_file($u);
     }
 }
 
-if ( !$parse ) {
+if (!$parse) {
     exit 0;
 }
 
-my $v4_fh;
-my $v6_fh;
-
-if ($v4) {
-    open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!";
-    print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
-    foreach my $ns ( @{$ns_servers} ) {
-        print $v4_fh "\$NS 43200 $ns\n";
-    }
-}
-if ($v6) {
-    open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!";
-    print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
-    foreach my $ns ( @{$ns_servers} ) {
-        print $v6_fh "\$NS 43200 $ns\n";
-    }
-}
-
-sub is_bougus_asn {
-    my $as = shift;
-
-    # 64496-64511 Reserved for use in documentation and sample code
-    # 64512-65534 Designated for private use
-    # 65535       Reserved
-    # 65536-65551 Reserved for use in documentation and sample code
-    # 65552-131071 Reserved
-    return 1 if $as >= 64496 && $as <= 131071;
-
-    # 4294967295
-    return 1 if $as == 4294967295;
+# Prefix to ASN map
+my $networks = { 4 => {}, 6 => {} };
 
-    # AS0 is reserved
-    # AS1 is legal AS, but in most cases used by others without permission
-    # of owner (probably lame admins use AS1 as private AS).
-    return 1 if $as <= 1;
+foreach my $u (@{ $config{'bgp_sources'} }) {
+    my $parsed = URI->new($u);
+    my $fname  = $download_target . '/' . basename($parsed->path);
 
-    return 0;
-}
+    use constant {
+      F_MARKER    => 0,
+      F_TIMESTAMP => 1,
+      F_PEER_IP   => 3,
+      F_PEER_AS   => 4,
+      F_PREFIX    => 5,
+      F_AS_PATH   => 6,
+      F_ORIGIN    => 7,
+    };
 
-# Now load BGP data
-my $networks = {};
+    open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";
 
-foreach my $u ( @{ $config{'bgp_sources'} } ) {
-    my $parsed = URI->new($u);
-    my $fname  = $download_target . '/' . basename( $parsed->path );
-
-    if ($bgpdump_path) {
-        use constant {
-            F_MARKER    => 0,
-            F_TIMESTAMP => 1,
-            F_PEER_IP   => 3,
-            F_PEER_AS   => 4,
-            F_PREFIX    => 5,
-            F_AS_PATH   => 6,
-            F_ORIGIN    => 7,
-        };
-
-        open( my $bgpd, '-|', "$bgpdump_path -v -M $fname" ) or die "can't start bgpdump: $!";
-
-        while (<$bgpd>) {
-            chomp;
-            my @e = split /\|/;
-            if ( $e[F_MARKER] ne 'TABLE_DUMP2' ) {
-                warn "bad line: $_\n";
-                next;
-            }
+    while (<$bgpd>) {
+        chomp;
+        my @e = split /\|/;
+        if ($e[F_MARKER] ne 'TABLE_DUMP2') {
+            warn "bad line: $_\n";
+            next;
+        }
 
-            my $origin_as;
-            my $prefix = $e[F_PREFIX];
-            my $ipv6   = 0;
+        my $origin_as;
+        my $prefix = $e[F_PREFIX];
+        my $ip_ver = 6;
 
-            if ( $prefix !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/ ) {
-                $ipv6 = 1;
-            }
+        if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
+            $ip_ver = 4;
+        }
 
-            if ( $e[F_AS_PATH] ) {
+        if ($e[F_AS_PATH]) {
 
-                # not empty AS_PATH
-                my @as_path = split /\s/, $e[F_AS_PATH];
-                $origin_as = pop @as_path;
+            # not empty AS_PATH
+            my @as_path = split /\s/, $e[F_AS_PATH];
+            $origin_as = pop @as_path;
 
-                if ( substr( $origin_as, 0, 1 ) eq '{' ) {
+            if (substr($origin_as, 0, 1) eq '{') {
 
-                    # route is aggregated
-                    if ( $origin_as =~ /^{(\d+)}$/ ) {
+                # route is aggregated
+                if ($origin_as =~ /^{(\d+)}$/) {
 
-                        # single AS aggregated, just remove { } around
-                        $origin_as = $1;
-                    }
-                    else {
-                        # use previous AS from AS_PATH
-                        $origin_as = pop @as_path;
-                    }
-                }
+                    # single AS aggregated, just remove { } around
+                    $origin_as = $1;
+                } else {
 
-                # strip bogus AS
-                while ( is_bougus_asn($origin_as) ) {
+                    # use previous AS from AS_PATH
                     $origin_as = pop @as_path;
-                    last if scalar @as_path == 0;
                 }
             }
 
-            # empty AS_PATH or all AS_PATH elements is stripped as bogus - use PEER_AS is origin AS
-            $origin_as //= $e[F_PEER_AS];
-
-            if ( !$networks->{$origin_as} ) {
-                if ( !$ipv6 ) {
-                    $networks->{$origin_as} = { nets_v4 => [$prefix], nets_v6 => [] };
-                }
-                else {
-                    $networks->{$origin_as} = { nets_v6 => [$prefix], nets_v4 => [] };
-                }
-            }
-            else {
-                if ( !$ipv6 ) {
-                    push @{ $networks->{$origin_as}->{'nets_v4'} }, $prefix;
-                }
-                else {
-                    push @{ $networks->{$origin_as}->{'nets_v6'} }, $prefix;
-                }
-            }
-        }
-    }
-    else {
-        require Net::MRT;
-        $Net::MRT::USE_RFC4760 = -1;
-
-        open( my $fh, "<:gzip", $fname )
-          or die "Cannot open $fname: $!";
-        while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) {
-            if ( $dd->{'prefix'} && $dd->{'bits'} ) {
-                next if $dd->{'subtype'} == 2 and !$v4;
-                next if $dd->{'subtype'} == 4 and !$v6;
-                my $entry = $dd->{'entries'}->[0];
-                my $net   = $dd->{'prefix'} . '/' . $dd->{'bits'};
-                if ( $entry && $entry->{'AS_PATH'} ) {
-                    my $as = $entry->{'AS_PATH'}->[-1];
-                    if ( ref($as) eq "ARRAY" ) {
-                        $as = @{$as}[0];
-                    }
-
-                    next if ( is_bougus_asn($as) );
-
-                    if ( !$networks->{$as} ) {
-                        if ( $dd->{'subtype'} == 2 ) {
-                            $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] };
-                        }
-                        else {
-                            $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] };
-                        }
-                    }
-                    else {
-
-                        if ( $dd->{'subtype'} == 2 ) {
-                            push @{ $networks->{$as}->{'nets_v4'} }, $net;
-                        }
-                        else {
-                            push @{ $networks->{$as}->{'nets_v6'} }, $net;
-                        }
-                    }
-                }
+            # strip bogus AS
+            while (is_bougus_asn($origin_as)) {
+                $origin_as = pop @as_path;
+                last if scalar @as_path == 0;
             }
         }
+
+        # empty AS_PATH or all AS_PATH elements was stripped as bogus - use
+        # PEER_AS as origin AS
+        $origin_as //= $e[F_PEER_AS];
+
+        $networks->{$ip_ver}{$prefix} = int($origin_as);
     }
 }
 
+# Remove default routes
+delete $networks->{4}{'0.0.0.0/0'};
+delete $networks->{6}{'::/0'};
+
 # Now roughly detect countries
-foreach my $u ( @{ $config{'asn_sources'} } ) {
+my $as_info = {};
+
+# RIR statistics exchange format
+# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
+# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
+# first 7 fields for this two formats are same
+use constant {
+  F_REGISTRY => 0,  # {afrinic,apnic,arin,iana,lacnic,ripencc}
+  F_CC       => 1,  # ISO 3166 2-letter contry code
+  F_TYPE     => 2,  # {asn,ipv4,ipv6}
+  F_START    => 3,
+  F_VALUE    => 4,
+  F_DATE     => 5,
+  F_STATUS   => 6,
+};
+
+foreach my $u (@{ $config{'asn_sources'} }) {
     my $parsed = URI->new($u);
-    my $fname  = $download_target . '/' . basename( $parsed->path );
-    open( my $fh, "<", $fname ) or die "Cannot open $fname: $!";
+    my $fname  = $download_target . '/' . basename($parsed->path);
+    open(my $fh, "<", $fname) or die "Cannot open $fname: $!";
 
     while (<$fh>) {
         next if /^\#/;
         chomp;
         my @elts = split /\|/;
 
-        if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) {
-            my $as_start = int( $elts[3] );
-            my $as_end   = $as_start + int( $elts[4] );
-
-            for ( my $as = $as_start ; $as < $as_end ; $as++ ) {
-                my $real_as = $as;
+        if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
+            my $as_start = int($elts[F_START]);
+            my $as_end   = $as_start + int($elts[F_VALUE]) - 1;
 
-                if ( ref($as) eq "ARRAY" ) {
-                    $real_as = @{$as}[0];
-                }
-
-                if ( $networks->{"$real_as"} ) {
-                    $networks->{"$real_as"}->{'country'} = $elts[1];
-                    $networks->{"$real_as"}->{'rir'}     = $elts[0];
-                }
+            for my $as ($as_start .. $as_end) {
+                $as_info->{$as}{'country'} = $elts[F_CC];
+                $as_info->{$as}{'rir'}     = $elts[F_REGISTRY];
             }
         }
     }
 }
 
-while ( my ( $k, $v ) = each( %{$networks} ) ) {
-    if ($v4) {
-        foreach my $n ( @{ $v->{'nets_v4'} } ) {
+# Write zone files
+my $ns_list     = join ' ', @{$ns_servers};
+my $zone_header = << "EOH";
+\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
+\$NS  43200 $ns_list
+EOH
 
-            # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
-            if ( $v->{'country'} ) {
-                printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
-            }
-            else {
-                printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
-            }
-        }
+if ($v4) {
+    open my $v4_fh, '>', ".$v4_file.tmp";
+    print $v4_fh $zone_header;
+
+    while (my ($net, $asn) = each %{ $networks->{4} }) {
+        my $country = $as_info->{$asn}{'country'} || 'UN';
+        my $rir     = $as_info->{$asn}{'rir'}     || 'UN';
+
+        # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8
+        printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
     }
-    if ($v6) {
-        foreach my $n ( @{ $v->{'nets_v6'} } ) {
 
-            # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
-            if ( $v->{'country'} ) {
-                printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
-            }
-            else {
-                printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
-            }
-        }
+    close $v4_fh;
+    rename ".$v4_file.tmp", $v4_file;
+}
+
+if ($v6) {
+    open my $v6_fh, '>', ".$v6_file.tmp";
+    print $v6_fh $zone_header;
+
+    while (my ($net, $asn) = each %{ $networks->{6} }) {
+        my $country = $as_info->{$asn}{'country'} || 'UN';
+        my $rir     = $as_info->{$asn}{'rir'}     || 'UN';
+
+        # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
+        printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
     }
+
+    close $v6_fh;
+    rename ".$v6_file.tmp", $v6_file;
+}
+
+exit 0;
+
+########################################################################
+
+sub download_file {
+    my ($url) = @_;
+
+    local $File::Fetch::WARN    = 0;
+    local $File::Fetch::TIMEOUT = 180;  # connectivity to ftp.lacnic.net is bad
+
+    my $ff    = File::Fetch->new(uri => $url);
+    my $where = $ff->fetch(to => $download_target) or
+      die "$url: ", $ff->error;
+
+    return $where;
+}
+
+# Returns true if AS number is bogus
+# e. g. a private AS.
+# List of allocated and reserved AS:
+# https://www.iana.org/assignments/as-numbers/as-numbers.txt
+sub is_bougus_asn {
+    my $as = shift;
+
+    # 64496-64511  Reserved for use in documentation and sample code
+    # 64512-65534  Designated for private use
+    # 65535        Reserved
+    # 65536-65551  Reserved for use in documentation and sample code
+    # 65552-131071 Reserved
+    return 1 if $as >= 64496 && $as <= 131071;
+
+    # Reserved (RFC6996, RFC7300, RFC7607)
+    return 1 if $as == 0 || $as >= 4200000000;
+
+    return 0;
 }
 
 __END__
@@ -324,8 +278,8 @@ asn.pl - download and parse ASN data for Rspamd
 asn.pl [options]
 
  Options:
-   --download-asn         Download ASN data from RIR
-   --download-bgp         Download GeoIP data from Maxmind
+   --download-asn         Download ASN data from RIRs
+   --download-bgp         Download BGP full view dump from RIPE RIS
    --target               Where to download files (default: current dir)
    --zone-v4              IPv4 zone (default: asn.rspamd.com)
    --zone-v6              IPv6 zone (default: asn6.rspamd.com)
@@ -365,3 +319,5 @@ Prints the manual page and exits.
 B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.
 
 =cut
+
+# vim: et:ts=4:sw=4
\ No newline at end of file