From a134799dddb07dca45196ccc9737415d2cc5b586 Mon Sep 17 00:00:00 2001
From: korgoth1 <vladislav.stakhov@gmail.com>
Date: Wed, 15 Jan 2020 23:35:46 +0300
Subject: [PATCH] [Test] SPF external IP map

---
 conf/maps.d/external_relay_ip.inc  |   1 +
 conf/maps.d/surbl-whitelist.inc    | 831 +----------------------------
 conf/modules.d/spf.conf            |   6 +
 src/plugins/lua/spf.lua            |  26 +-
 test/functional/configs/dmarc.conf |   4 +-
 5 files changed, 35 insertions(+), 833 deletions(-)
 create mode 100644 conf/maps.d/external_relay_ip.inc

diff --git a/conf/maps.d/external_relay_ip.inc b/conf/maps.d/external_relay_ip.inc
new file mode 100644
index 000000000..3fc5c1775
--- /dev/null
+++ b/conf/maps.d/external_relay_ip.inc
@@ -0,0 +1 @@
+192.168.1.1
\ No newline at end of file
diff --git a/conf/maps.d/surbl-whitelist.inc b/conf/maps.d/surbl-whitelist.inc
index b0efda20a..3fc5c1775 100644
--- a/conf/maps.d/surbl-whitelist.inc
+++ b/conf/maps.d/surbl-whitelist.inc
@@ -1,830 +1 @@
-126.com
-163.com
-1gost.info
-1stnationalbank.com
-2o7.net
-365online.com
-4at1.com
-53.com
-5iantlavalamp.com
-abl.com.pk
-about.com
-accessbankplc.com
-adelphia.net
-adib.ae
-adobe.com
-agora-inc.com
-agoramedia.com
-aibgb.co.uk
-aib.ie
-airdriesavingsbank.com
-akamai.net
-akamaitech.net
-aldermore.co.uk
-alexa.com
-alliancebank.com.my
-alliancefg.com
-alliantcreditunion.com
-alliantcreditunion.org
-allianz.de
-allybank.com
-alterna.ca
-amazon.com
-americanexpress.ch
-americanexpress.com
-anadolubank.nl
-ancestry.com
-anz.com
-anz.co.nz
-aol.com
-apache.org
-apple.com
-arbuthnotlatham.co.uk
-arcamax.com
-asb.co.nz
-ask.com
-astrology.com
-atdmt.com
-att.net
-authorize.net
-autorambler.ru
-axisbank.co.in
-axisbank.com
-b2bbank.com
-baaderbank.de
-baidu.com
-baloise.ch
-baml.com
-banamex.com
-bancanetbsc.do
-bancanetsantacruz.com.do
-bancapulia.it
-bancarios.com
-bancastato.ch
-bancatransilvania.ro
-bancobase.com
-bancobic.ao
-bancobic.pt
-bancobpi.pt
-banco.bradesco
-bancobrasil.com.br
-bancochile.cl
-bancochile.com
-bancoestado.cl
-bancofalabella.cl
-bancofalabella.com.co
-bancofalabella.pe
-bancomer.com
-bancopopolare.it
-bancopostaclick.it
-bancoposta.it
-bancosantander.es
-bancovotorantimcartoes.com.br
-bank24.ru
-bankalhabib.com
-bankaustria.at
-bank.barclays.co.uk
-bankbgzbnpparibas.pl
-bankcardservices.co.uk
-bankcomm.com
-bankcoop.ch
-bankiabancapersonal.es
-bankia.com
-bankia.es
-bankinter.com
-bankinter.es
-bankmutual.com
-bankofamerica.com
-bankofcanada.ca
-bankofchina.com
-bankofcyprus.com
-bankofindia.co.nz
-bankofireland.com
-bank-of-ireland.co.uk
-bankofirelanduk.com
-bankofoklahoma.com
-bankofscotland.co.uk
-banksinarmas.com
-bankvonroll.ch
-bankwest.com.au
-banque-casino.fr
-banquepopulaire.fr
-banquescotia.com
-barclaycard.co.uk
-barclaycard.de
-barclaycard.es
-barclays.com
-barclays.co.uk
-barclayspartnerfinance.com
-barclays.sc
-barodanzltd.co.nz
-basler.ch
-bbandt.com
-bbc.co.uk
-bcentral.com
-bci.cl
-bcp.com.pe
-bcv.ch
-bcvs.ch
-bekb.ch
-bellevue.ch
-bellsouth.net
-bendigobank.com.au
-berliner-bank.de
-berliner-sparkasse.de
-bfanet.ao
-bfi0.com
-bgfi.com
-bgfionline.com
-bgzbnpparibas.pl
-billmelater.com
-bing.com
-bkb.ch
-bk.rw
-bks.at
-blkb.ch
-bmocm.com
-bmo.com
-bmogam.com
-bmoharris.com
-bmoharrisprivatebankingonline.com
-bmoinvestorline.com
-bmonesbittburns.com
-bnl.it
-bnpparibas.com
-bnpparibas.fr
-boc.cnnz
-bonuscard.ch
-bpe-gruposantander.com
-bpi.pt
-bpostbank.be
-bradescardonline.com.br
-bradesco.com.br
-bradescoseguranca.com.br
-bridgetrack.com
-bridgewaterbank.ca
-bsibank.com
-btrl.ro
-bt-trade.ro
-businessonline-boi.com
-bzbank.ch
-ca-cib.com
-ca-egypt.com
-cafbank.org
-cafe24.com
-cafonline.org
-caisse-epargne.com
-caisse-epargne.fr
-caixabank.com
-caixa.gov.br
-cajasur.es
-camsonline.com
-canadiandirect.com
-capitalone360.com
-capitalone.com
-capitaloneonline.co.uk
-capitecbank.co.za
-cariparma.it
-carrefour-banque.fr
-cartabcc.it
-cartabccpos.it
-cartasi.it
-ca-suisse.com
-catalunyacaixa.com
-cbg.gm
-cbonline.co.uk
-cembra.ch
-cenbank.org
-centralbank.ae
-charitybank.org
-charter.net
-chase.com
-chebanca.it
-chinatrust.com.tw
-cial.ch
-cibc.com
-cic.ch
-cimbclicks.com.my
-citibank.ae
-citibank.co.in
-citibank.com
-citibank.co.uk
-citibankonline.com
-citibusiness.com
-citicards.com
-citi.com
-citi.co.nz
-citi.eu
-citigroup.com
-citizensbank.ca
-citizensbank.com
-civibank.com
-civibank.it
-cjb.net
-classmates.com
-clickbank.net
-closebrothers.com
-closebrothers.co.uk
-clubsc.ch
-cnet.com
-cnn.com
-colpatria.com
-colpatria.com.co
-comcast.net
-com.com
-commbank.com.au
-commerzbank.com
-commerzbank.de
-com.ne.kr
-coopbank.dk
-co-operativebank.co.uk
-cornerbanca.ch
-cornercard.ch
-cornercard.com
-corner.ch
-corporate-ir.net
-cosycard.ch
-coutts.com
-cox.net
-craigslist.org
-credit-agricole.com
-credit-agricole.fr
-creditagricole.rs
-credit-suisse.com
-cs.com
-css.ch
-ctbcbank.com
-ctfs.com
-custhelp.com
-cwbank.com
-cwbankgroup.com
-cwt.ca
-cybg.com
-danskebankas.lt
-danskebank.com
-danskebank.co.uk
-danskebank.de
-danskebank.dk
-danskebank.ee
-danskebank.fi
-danskebank.ie
-danskebank.no
-datatrans.biz
-datatrans.ch
-daum.net
-db.com
-dbs.com
-dd.se
-debian.org
-dell.com
-demirbank.kg
-denizbank.com
-desjardins.ca
-desjardins.com
-deutschebank.be
-deutschebank.co.nz
-deutsche-bank.de
-diamondbank.com
-dibpak.com
-directnic.com
-directtrack.com
-discovercard.com
-discover.com
-discovery.co.za
-dnbnord.lt
-domain.com
-doubleclick.com
-dresdner-bank.de
-dsbbank.sr
-dsbl.org
-duncanlawrie.com
-earthlink.net
-easybank.at
-easylnk.com
-ebay.com
-ebay.co.uk
-ebay.de
-ebayimg.com
-ebaystatic.com
-ecobank.com
-edgesuite.net
-ediets.com
-edwardjones.com
-egroups.com
-e-gulfbank.com
-emode.com
-esunbank.com.tw
-example.com
-example.net
-example.org
-excite.com
-facebook.com
-fedex.com
-fednetbank.com
-fidelity.com
-fidor.de
-finance.com
-finansbank.com.tr
-finasta.lt
-fineco.it
-firstbankcard.com
-firstmerit.com
-firstnational.com
-firstnationalmerchantsolutions.com
-firsttrustbank.co.uk
-flickr.com
-fnbc.ca
-fnb.co.za
-fnb-online.com
-freebsd.org
-free.fr
-friuladria.it
-f-secure.com
-garantibank.eu
-garantibank.nl
-garanti.com.tr
-gazprombank.ch
-gazprombank.ru
-generali.es
-genevoise.ch
-gentoo.org
-geocities.com
-gkb.ch
-gmail.com
-gmx.net
-go.com
-godaddy.com
-googleadservices.com
-google.co.in
-google.com
-google.it
-google.ru
-granitbank.hu
-grisoft.com
-gtbank.com
-halifax.co.uk
-hallmark.com
-handelsbanken.se
-harrodsbank.co.uk
-hbl.com
-hblibank.com
-hblibank.com.pk
-hdfcbank.com
-heartland.co.nz
-hellenicbank.com
-hinet.net
-hkbea.com
-hlb.com.kh
-hlb.com.my
-hoaresbank.co.uk
-home.barclays
-hongleongconnect.com.kh
-hongleongconnect.com.vn
-hongleongconnect.my
-hotbar.com
-hotmail.com
-hotpop.com
-hp.com
-hsbc.com
-hsbc.com.ar
-hsbc.com.hk
-hsbc.co.nz
-hsbc.co.uk
-hypovereinsbank.co.uk
-hypovereinsbank.de
-ibm.com
-icbcnz.com
-icicibank.co.in
-icicibank.com
-icicibankprivatebanking.com
-icorner.ch
-icscards.de
-icscards.nl
-incredimail.com
-ing.be
-ing.com
-ing-diba.de
-ingdirect.ca
-ing.lu
-ing.nl
-ingvysyabank.com
-interac.ca
-investorplace.com
-iobnet.co.in
-isbank.com.tr
-isbank.de
-isbank.ge
-isbank.iq
-isbankkosova.com
-itau.com.br
-ivillage.com
-joingevalia.com
-jpmchase.com
-jpmorgan.com
-jsafrasarasin.com
-julianhodgebank.com
-juliusbaer.com
-juno.com
-jyskebank.dk
-kantonalbank.ch
-kernel.org
-key.com
-kiwibank.co.nz
-kotak.com
-kredytbank.pl
-kreissparkasse-schwalm-eder.de
-ksklb.de
-kutxabank.es
-laboralkutxa.com
-lacaixa.cat
-lacaixa.es
-laurentianbank.ca
-lbb.de
-lcl.com
-lcl.fr
-li.ru
-list.ru
-liveinternet.ru
-livejournal.com
-lloydsbank.com
-lloydsbankcommercial.com
-lloydsbankinggroup.com
-lloydstsb.ch
-lloydstsb.co.uk
-lombardodier.com
-loydsbank.com
-lycos.com
-m7z.net
-mac.com
-macromedia.com
-maerki-baumann.ch
-mail.com
-mail.ru
-mailscanner.info
-mandtbank.com
-manulifebank.ca
-manulifebankselect.ca
-manulife.com
-manulifeone.ca
-marketwatch.com
-mashreqbank.com
-mastercard.com
-maybank2u.com
-maybank2u.com.my
-mcafee.com
-mchsi.com
-mdmbank.com
-mechanicsbank.com
-medbank.lt
-messagelabs.com
-metrobankdirect.com
-metrobankonline.co.uk
-microsoft.com
-migbank.com
-migrosbank.ch
-military.com
-mindspring.com
-mit.edu
-mizuhobank.co.jp
-mmwarburg.lu
-monster.com
-montepio.pt
-morganstanley.com
-mozilla.com
-mps.it
-ms.com
-msn.com
-mufg.jp
-myonlineresourcecenter.com
-myonlineservices.ch
-myspace.com
-nate.com
-nationalesuisse.ch
-nationwide-communications.co.uk
-nationwide.co.uk
-nationwide-service.co.uk
-natwest.com
-navyfederal.org
-nbc.ca
-netflix.com
-netscape.com
-netscape.net
-netzero.net
-newyorkfed.org
-nibl.com.np
-nod32.com
-nordea.fi
-nordea.lt
-nordfynsbank.dk
-norisbank.de
-norman.com
-notenstein.ch
-nuvisionfederal.com
-nytimes.com
-oceanbank.com
-onlinesbi.com
-openoffice.org
-openxmlformats.org
-optonline.net
-orchardbank.com
-osdn.com
-ostsaechsische-sparkasse-dresden.de
-overstock.com
-pacbell.net
-pandasoftware.com
-passport.com
-paylife.at
-paypal.be
-paypal-brasil.com.br
-paypal.ca
-paypal.ch
-paypal.co.il
-paypal.com
-paypal.com.au
-paypal.com.br
-paypal-communication.com
-paypal-community.com
-paypal.com.mx
-paypal.com.pt
-paypal.co.uk
-paypal-customerfeedback.com
-paypal.de
-paypal-deutschland.de
-paypal.dk
-paypal.es
-paypal-exchanges.com
-paypal.fr
-paypal.it
-paypal-marketing.co.uk
-paypal-marketing.pl
-paypal.net
-paypal.nl
-paypal.no
-paypal-notify.com
-paypal-now.com
-paypalobjects.com
-paypal-opwaarderen.nl
-paypal-pages.com
-paypal.pt
-paypal.ru
-paypal.se
-paypal-search.com
-paypal-shopping.co.uk
-paypal-techsupport.com
-pbebank.com
-pcfinancial.ca
-peoplepc.com
-permanenttsb.ie
-plaxo.com
-pnc.com
-popolarevicenza.it
-postbank.de
-postepay.it
-postfinancearena.ch
-postfinance.ch
-postfinance.info
-price.ru
-prodigy.net
-publicislamicbank.com.my
-rabobank.com
-rabobank.co.nz
-rabobank.nl
-radaruol.com.br
-rahnbodmer.ch
-raiffeisenbank.rs
-raiffeisen.ch
-raiffeisen.hu
-raiffeisen.li
-raiffeisen.ru
-rambler-co.ru
-rambler.ru
-raphaelsbank.com
-rbc.com
-rbcroyalbank.com
-rbs.co.uk
-rbssecure.co.uk
-rbsworldpay.com
-rcb.at rcb.at
-real.com
-recordbank.be
-redhat.com
-rediff.com
-regiobank.nl
-regions.com
-regionsnet.com
-renasantbank.com
-rhbgroup.com
-rogersbank.com
-rogers.com
-rothschildbank.com
-rothschild.com
-royalbank.com
-rr.com
-sagepay.com
-sagepay.co.uk
-sainsburysbank.co.uk
-samba.com
-santander.cl
-santander.com
-santander.com.br
-santander.com.mx
-santandercorretora.com.br
-santander.co.uk
-santanderesfera.com.br
-santandersantiago.cl
-sarasin.ch
-sbcglobal.net
-sberbank.ch
-sbs.net.nz
-sc.com
-schoellerbank.at
-scotiabank.ca
-scotiabank.com
-scotiamocatta.com
-scotiaonline.com
-s.de
-sec.gov
-securetrustbank.com
-service-sparkasse.de
-serviciobancomer.com
-sf.net
-shawbrook.co.uk
-shaw.ca
-shkb.ch
-shockwave.com
-six-group.com
-six-payment-services.com
-skrill.com
-sls-direkt.de
-smithbarney.com
-snb.ch snb.ch
-snsbank.nl
-societegenerale.fr
-sourceforge.net
-spamcop.net
-sparda-a.de
-sparda-bank-hamburg.de
-sparda-b.de
-sparda-bw.de
-sparda-h.de
-sparda-hessen.de
-sparda-m.de
-sparda-ms.de
-sparda-n.de
-sparda-ostbayern.de
-sparda-sw.de
-sparda-verband.de
-sparda-west.de
-sparkasse.at
-sparkasse-bank-malta.com
-sparkasse-bielefeld.de
-sparkasseblog.de
-sparkasse-bochum.de
-sparkasse.ch
-sparkasse.de
-sparkasse-gera-greiz.de
-sparkasse-hamm.de
-sparkasse-heidelberg.de
-sparkasse-ingolstadt.de
-sparkasse-mittelthueringen.de
-speedera.net
-sportsline.com
-standardbank.com
-standardbank.co.za
-standardchartered.com.gh
-standardchartered.com.my
-subscribe.ru
-sun.com
-suncorpbank.com.au
-suntrust.com
-swedbank.com
-swedbank.ee
-swedbank.lt
-swedbank.lu
-swedbank.se
-swisscanto.ch
-swisscaution.ch
-swissquote.ch
-sydbank.dk
-sympatico.ca
-tails.nl
-tangerine.ca
-tcb-bank.com.tw
-tdbank.com
-tdcommercialbanking.com
-telus.net
-terra.com.br
-tescobank.com
-ticketmaster.com
-tinyurl.com
-tiscali.co.uk
-tns-counter.ru
-tom.com
-tone.co.nz
-t-online.de
-top4top.ru
-tsbbank.co.nz
-tsb.co.nz
-tsb.co.uk
-tux.org
-twitter.com
-ubibanca.com
-ubs.com
-ulsterbankanytimebanking.co.uk
-ulsterbank.co.uk
-unibancoconnect.pt
-unibanco.pt
-unicreditbank.lt
-unicredit.eu
-unicreditgroup.eu
-unicredit.it
-unionbankcameroon.com
-unionbank.com
-unity.co.uk
-uob.com.sg
-uobgroup.com
-uol.com.br
-ups.com
-usbank.com
-valianttrust.com
-vaudoise.ch
-venetobanca.it
-venetobanka.al
-verizon.net
-versabank.com
-videobank.it
-virginmoney.com
-visa.com.ar
-visa.com.br
-visaeurope.ch
-visaeurope.com
-viseca.ch
-volksbank.de
-volkswagenbank.de
-vpbank.com
-vr.de
-vwbank.de
-w3.org
-wachovia.com
-walmart.com
-wamu.com
-wanadoo.fr
-washingtonpost.com
-weatherbug.com
-weatherbys.co.uk
-web.de
-webshots.com
-webtv.net
-wegelin.ch
-wellsfargo.com
-wellsfargoemail.com
-westernunion.ca
-westernunion.com
-westernunion.fr
-westernunion.se
-westpac.com.au
-westpac.co.nz
-wir.ch
-wordpress.com
-worldbank.org
-worldpay.com
-wsj.com
-wvb.de
-xmlsoap.org
-yacht.nl
-yahoo.ca
-yahoo.co.jp
-yahoo.co.kr
-yahoo.com
-yahoo.com.br
-yahoo.co.uk
-yahoogroups.com
-yandex.net
-yandex.ru
-ybonline.co.uk
-yimg.com
-yopi.de
-yorkshirebank.co.uk
-yourbankcard.com
-yoursite.com
-youtube.com
-zagbank.ca
-zdnet.com
-zenithbank.com
-zkb.ch
-zugerkb.ch
-vistaprint.dk
-vistaprint.com
-anpdm.com
-dovecot.org
-exacttarget.com
-github.com
-isc.org
-# list-manage1.com # grey
-# list-manage2.com # grey
-# list-manage.com # grey
-lists.isc.org
-lists.roundcube.net
-svn.apache.org
-taggedmail.com
-tumblr.com
+192.168.1.1
\ No newline at end of file
diff --git a/conf/modules.d/spf.conf b/conf/modules.d/spf.conf
index 5dddecf97..cf0862909 100644
--- a/conf/modules.d/spf.conf
+++ b/conf/modules.d/spf.conf
@@ -18,4 +18,10 @@ spf {
   .include(try=true,priority=5) "${DBDIR}/dynamic/spf.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/spf.conf"
   .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/spf.conf"
+
+  external_relay = [
+    "$LOCAL_CONFDIR/local.d/maps.d/external_relay_ip.inc.local",
+    "${DBDIR}/surbl-external_relay_ip.inc.local",
+    "fallback+file://${CONFDIR}/maps.d/external_relay_ip.inc"
+  ];
 }
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index 10daa0d2b..3dbef56a5 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -67,6 +67,7 @@ local default_config = {
   disable_ipv6 = false,
   symbols = symbols,
   external_relay = nil,
+  external_map = nil,
 }
 
 local local_config = rspamd_config:get_all_opt('spf')
@@ -89,7 +90,8 @@ local function spf_check_callback(task)
     local found = false
 
     for i,hdr in ipairs(rh) do
-      if hdr.real_ip and hdr.real_ip == local_config.external_relay then
+      --if hdr.real_ip and hdr.real_ip == local_config.external_relay then
+      if hdr.real_ip and local_config.external_relay:get_key(hdr.real_ip) then
         -- We can use the next header as a source of IP address
         if rh[i + 1] then
           local nhdr = rh[i + 1]
@@ -218,7 +220,7 @@ if local_config.whitelist then
       "radix", "SPF whitelist map")
 end
 
-if local_config.external_relay then
+--[[if local_config.external_relay then
   local rspamd_ip = require "rspamd_ip"
   local ip = rspamd_ip.from_string(local_config.external_relay)
 
@@ -228,9 +230,29 @@ if local_config.external_relay then
     local_config.external_relay = nil
   else
     local_config.external_relay = ip
+  end]]
+
+local function external_ip_spf(task)
+  if local_config.external_relay then
+    if local_config.external_relay:get_key(task:get_received_headers()) then
+      task:insert_result('EXT_RELAY', 1.0, task:get_received_headers())
+    end
   end
 end
 
+if local_config.external_relay then
+  local lua_maps = require "lua_maps"
+
+  local_config.external_relay = lua_maps.map_add_from_ucl(local_config.external_relay,
+    mtype: 'radix', description: 'External IP SPF map')
+end
+
+rspamd_config:register_symbol{
+  type = 'normal'
+  callback = external_ip_spf,
+  name = 'EXT_RELAY'
+}
+
 for _,sym in pairs(local_config.symbols) do
   rspamd_config:register_symbol{
     name = sym,
diff --git a/test/functional/configs/dmarc.conf b/test/functional/configs/dmarc.conf
index 08a542c70..f086f9238 100644
--- a/test/functional/configs/dmarc.conf
+++ b/test/functional/configs/dmarc.conf
@@ -1,4 +1,6 @@
 dmarc { }
 spf {
-	external_relay = 192.168.1.1;
+  external_relay = [
+    "$LOCAL_CONFDIR/local.d/maps.d/external_relay_ip.inc.local",
+  ];
 }
\ No newline at end of file
-- 
2.39.5