123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331 |
- #!/usr/bin/env perl
- #
-
- use warnings;
- use strict;
- use autodie;
-
- use File::Basename;
- use File::Fetch;
- use Getopt::Long;
- use Pod::Usage;
-
- use FindBin;
- use lib "$FindBin::Bin/extlib/lib/perl5";
-
- use URI;
-
- my %config = (
- asn_sources => [
- 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
- 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
- 'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
- 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
- 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
- ],
- bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
- );
-
- my $download_asn = 0;
- my $download_bgp = 0;
- my $download_target = "./";
- my $help = 0;
- my $man = 0;
- my $v4 = 1;
- my $v6 = 1;
- my $parse = 1;
- my $v4_zone = "asn.rspamd.com";
- my $v6_zone = "asn6.rspamd.com";
- my $v4_file = "asn.zone";
- my $v6_file = "asn6.zone";
- my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
- my $unknown_placeholder = "--";
-
- GetOptions(
- "download-asn" => \$download_asn,
- "download-bgp" => \$download_bgp,
- "4!" => \$v4,
- "6!" => \$v6,
- "parse!" => \$parse,
- "target=s" => \$download_target,
- "zone-v4=s" => \$v4_zone,
- "zone-v6=s" => \$v6_zone,
- "file-v4=s" => \$v4_file,
- "file-v6=s" => \$v6_file,
- "ns-server=s@" => \$ns_servers,
- "help|?" => \$help,
- "man" => \$man,
- "unknown-placeholder" => \$unknown_placeholder,
- ) or
- pod2usage(2);
-
- pod2usage(1) if $help;
- pod2usage(-exitval => 0, -verbose => 2) if $man;
-
- if ($download_asn) {
- foreach my $u (@{ $config{'asn_sources'} }) {
- download_file($u);
- }
- }
-
- if ($download_bgp) {
- foreach my $u (@{ $config{'bgp_sources'} }) {
- download_file($u);
- }
- }
-
- if (!$parse) {
- exit 0;
- }
-
- # Prefix to ASN map
- my $networks = { 4 => {}, 6 => {} };
-
- foreach my $u (@{ $config{'bgp_sources'} }) {
- my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename($parsed->path);
-
- use constant {
- F_MARKER => 0,
- F_TIMESTAMP => 1,
- F_PEER_IP => 3,
- F_PEER_AS => 4,
- F_PREFIX => 5,
- F_AS_PATH => 6,
- F_ORIGIN => 7,
- };
-
- open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";
-
- while (<$bgpd>) {
- chomp;
- my @e = split /\|/;
- if ($e[F_MARKER] ne 'TABLE_DUMP2') {
- warn "bad line: $_\n";
- next;
- }
-
- my $origin_as;
- my $prefix = $e[F_PREFIX];
- my $ip_ver = 6;
-
- if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
- $ip_ver = 4;
- }
-
- if ($e[F_AS_PATH]) {
-
- # not empty AS_PATH
- my @as_path = split /\s/, $e[F_AS_PATH];
- $origin_as = pop @as_path;
-
- if (substr($origin_as, 0, 1) eq '{') {
-
- # route is aggregated
- if ($origin_as =~ /^{(\d+)}$/) {
-
- # single AS aggregated, just remove { } around
- $origin_as = $1;
- } else {
-
- # use previous AS from AS_PATH
- $origin_as = pop @as_path;
- }
- }
-
- # strip bogus AS
- while (is_bougus_asn($origin_as)) {
- $origin_as = pop @as_path;
- last if scalar @as_path == 0;
- }
- }
-
- # empty AS_PATH or all AS_PATH elements was stripped as bogus - use
- # PEER_AS as origin AS
- $origin_as //= $e[F_PEER_AS];
-
- $networks->{$ip_ver}{$prefix} = int($origin_as);
- }
- }
-
- # Remove default routes
- delete $networks->{4}{'0.0.0.0/0'};
- delete $networks->{6}{'::/0'};
-
- # Now roughly detect countries
- my $as_info = {};
-
- # RIR statistics exchange format
- # https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
- # https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
- # first 7 fields for this two formats are same
- use constant {
- F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc}
- F_CC => 1, # ISO 3166 2-letter contry code
- F_TYPE => 2, # {asn,ipv4,ipv6}
- F_START => 3,
- F_VALUE => 4,
- F_DATE => 5,
- F_STATUS => 6,
- };
-
- foreach my $u (@{ $config{'asn_sources'} }) {
- my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename($parsed->path);
- open(my $fh, "<", $fname) or die "Cannot open $fname: $!";
-
- while (<$fh>) {
- next if /^\#/;
- chomp;
- my @elts = split /\|/;
-
- if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
- my $as_start = int($elts[F_START]);
- my $as_end = $as_start + int($elts[F_VALUE]) - 1;
-
- for my $as ($as_start .. $as_end) {
- $as_info->{$as}{'country'} = $elts[F_CC];
- $as_info->{$as}{'rir'} = $elts[F_REGISTRY];
- }
- }
- }
- }
-
- # Write zone files
- my $ns_list = join ' ', @{$ns_servers};
- my $zone_header = << "EOH";
- \$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
- \$NS 43200 $ns_list
- EOH
-
- if ($v4) {
- # create temp file in the same dir so we can be sure that mv is atomic
- my $out_dir = dirname($v4_file);
- my $out_file = basename($v4_file);
- my $temp_file = "$out_dir/.$out_file.tmp";
- open my $v4_fh, '>', $temp_file;
- print $v4_fh $zone_header;
-
- while (my ($net, $asn) = each %{ $networks->{4} }) {
- my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
- my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder;
-
- # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8
- printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
- }
-
- close $v4_fh;
- rename $temp_file, $v4_file;
- }
-
- if ($v6) {
- my $out_dir = dirname($v6_file);
- my $out_file = basename($v6_file);
- my $temp_file = "$out_dir/.$out_file.tmp";
- open my $v6_fh, '>', $temp_file;
- print $v6_fh $zone_header;
-
- while (my ($net, $asn) = each %{ $networks->{6} }) {
- my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
- my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder;
-
- # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
- printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
- }
-
- close $v6_fh;
- rename $temp_file, $v6_file;
- }
-
- exit 0;
-
- ########################################################################
-
- sub download_file {
- my ($url) = @_;
-
- local $File::Fetch::WARN = 0;
- local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad
-
- my $ff = File::Fetch->new(uri => $url);
- my $where = $ff->fetch(to => $download_target) or
- die "$url: ", $ff->error;
-
- return $where;
- }
-
- # Returns true if AS number is bogus
- # e. g. a private AS.
- # List of allocated and reserved AS:
- # https://www.iana.org/assignments/as-numbers/as-numbers.txt
- sub is_bougus_asn {
- my $as = shift;
-
- # 64496-64511 Reserved for use in documentation and sample code
- # 64512-65534 Designated for private use
- # 65535 Reserved
- # 65536-65551 Reserved for use in documentation and sample code
- # 65552-131071 Reserved
- return 1 if $as >= 64496 && $as <= 131071;
-
- # Reserved (RFC6996, RFC7300, RFC7607)
- return 1 if $as == 0 || $as >= 4200000000;
-
- return 0;
- }
-
- __END__
-
- =head1 NAME
-
- asn.pl - download and parse ASN data for Rspamd
-
- =head1 SYNOPSIS
-
- asn.pl [options]
-
- Options:
- --download-asn Download ASN data from RIRs
- --download-bgp Download BGP full view dump from RIPE RIS
- --target Where to download files (default: current dir)
- --zone-v4 IPv4 zone (default: asn.rspamd.com)
- --zone-v6 IPv6 zone (default: asn6.rspamd.com)
- --file-v4 IPv4 zone file (default: ./asn.zone)
- --file-v6 IPv6 zone (default: ./asn6.zone)
- --unknown-placeholder Placeholder for unknown elements (default: --)
- --help Brief help message
- --man Full documentation
-
- =head1 OPTIONS
-
- =over 8
-
- =item B<--download-asn>
-
- Download ASN data from RIR.
-
- =item B<--download-bgp>
-
- Download GeoIP data from Ripe
-
- =item B<--target>
-
- Specifies where to download files.
-
- =item B<--help>
-
- Print a brief help message and exits.
-
- =item B<--man>
-
- Prints the manual page and exits.
-
- =back
-
- =head1 DESCRIPTION
-
- B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.
-
- =cut
-
- # vim: et:ts=4:sw=4
|