From a5b96f1b715d8cf38822f8949cc9eb69a9b34966 Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Sat, 15 Dec 2018 14:44:25 +0000
Subject: [PATCH] [Rework] Rewrite RBL module

---
 src/plugins/lua/rbl.lua | 853 ++++++++++++++++++++--------------------
 1 file changed, 424 insertions(+), 429 deletions(-)

diff --git a/src/plugins/lua/rbl.lua b/src/plugins/lua/rbl.lua
index 32eb4f2d0..869f12f77 100644
--- a/src/plugins/lua/rbl.lua
+++ b/src/plugins/lua/rbl.lua
@@ -24,6 +24,7 @@ local rspamd_logger = require 'rspamd_logger'
 local rspamd_util = require 'rspamd_util'
 local fun = require 'fun'
 local lua_util = require 'lua_util'
+local ts = require("tableshape").types
 
 -- This plugin implements various types of RBL checks
 -- Documentation can be found here:
@@ -32,40 +33,49 @@ local lua_util = require 'lua_util'
 local E = {}
 local N = 'rbl'
 
-local rbls = {}
-local local_exclusions = nil
+local local_exclusions
 
 local default_monitored = '1.0.0.127'
 
 local function validate_dns(lstr)
   if lstr:match('%.%.') then
+    -- two dots in a row
     return false
   end
   for v in lstr:gmatch('[^%.]+') do
     if not v:match('^[%w-]+$') or v:len() > 63
       or v:match('^-') or v:match('-$') then
+      -- too long label or weird labels
       return false
     end
   end
   return true
 end
 
-local hash_alg = {
-  sha1 = true,
-  md5 = true,
-  sha256 = true,
-  sha384 = true,
-  sha512 = true,
-}
+local function maybe_make_hash(data, rule)
+  if rule.hash then
+    local h = hash.create_specific(rule.hash, data)
+    local s
+    if rule.hash_format then
+      if rule.hash_format == 'base32' then
+        s = h:base32()
+      elseif rule.hash_format == 'base64' then
+        s = h:base64()
+      else
+        s = h:hex()
+      end
+    else
+      s = h:hex()
+    end
+
+    if rule.hash_len then
+      s = s:sub(1, rule.hash_len)
+    end
 
-local function make_hash(data, specific)
-  local h
-  if not hash_alg[specific] then
-    h = hash.create(data)
+    return s
   else
-    h = hash.create_specific(specific, data)
+    return data
   end
-  return h:hex()
 end
 
 local function is_excluded_ip(rip)
@@ -75,8 +85,8 @@ local function is_excluded_ip(rip)
   return false
 end
 
-local function ip_to_rbl(ip, rbl)
-  return table.concat(ip:inversed_str_octets(), '.') .. '.' .. rbl
+local function ip_to_rbl(ip)
+  return table.concat(ip:inversed_str_octets(), '.')
 end
 
 local function gen_check_rcvd_conditions(rbl, received_total)
@@ -146,356 +156,304 @@ local function gen_check_rcvd_conditions(rbl, received_total)
   end
 end
 
-local function rbl_cb (task)
-  local function gen_rbl_callback(rule)
-    return function (_, to_resolve, results, err)
-      if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
-        rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err)
+local function rbl_dns_process(task, rbl, to_resolve, results, err)
+  if err and (err ~= 'requested record is not found' and
+      err ~= 'no records with this name') then
+    rspamd_logger.errx(task, 'error looking up %s: %s', to_resolve, err)
+  end
+  if not results then
+    lua_util.debugm(N, task,
+        'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+        to_resolve, false, err, rbl.symbol)
+    return
+  else
+    lua_util.debugm(N, task,
+        'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4',
+        to_resolve, true, err, rbl.symbol)
+  end
+
+  if rbl.returncodes == nil and rbl.symbol ~= nil then
+    task:insert_result(rbl.symbol, 1, to_resolve)
+    return
+  end
+  for _,result in pairs(results) do
+    local ipstr = result:to_string()
+    lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
+    local foundrc = false
+    -- Check return codes
+    for s,i in pairs(rbl.returncodes) do
+      for _,v in pairs(i) do
+        if string.find(ipstr, '^' .. v .. '$') then
+          foundrc = v
+          task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr)
+          break
+        end
       end
-      if not results then
-        lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', to_resolve, false, err, rule['rbls'][1]['symbol'])
-        return
+    end
+    if not foundrc then
+      if rbl.unknown and rbl.symbol then
+        task:insert_result(rbl.symbol, 1, to_resolve)
       else
-        lua_util.debugm(N, task, 'DNS RESPONSE: label=%1 results=%2 error=%3 rbl=%4', to_resolve, true, err, rule['rbls'][1]['symbol'])
+        rspamd_logger.errx(task, 'RBL %1 returned unknown result: %2',
+            rbl.rbl, ipstr)
       end
+    end
+  end
+end
 
-      for _,rbl in ipairs(rule.rbls) do
-        if rbl['returncodes'] == nil and rbl['symbol'] ~= nil then
-          task:insert_result(rbl['symbol'], 1, to_resolve)
-          return
-        end
-        for _,result in pairs(results) do
-          local ipstr = result:to_string()
-          local foundrc
-          lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
-          for s,i in pairs(rbl['returncodes']) do
-            if type(i) == 'string' then
-              if string.find(ipstr, '^' .. i .. '$') then
-                foundrc = i
-                task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr)
-                break
-              end
-            elseif type(i) == 'table' then
-              for _,v in pairs(i) do
-                if string.find(ipstr, '^' .. v .. '$') then
-                  foundrc = v
-                  task:insert_result(s, 1, to_resolve .. ' : ' .. ipstr)
-                  break
-                end
-              end
-            end
-          end
-          if not foundrc then
-            if rbl['unknown'] and rbl['symbol'] then
-              task:insert_result(rbl['symbol'], 1, to_resolve)
-            else
-              rspamd_logger.errx(task, 'RBL %1 returned unknown result: %2',
-                rbl['rbl'], ipstr)
-            end
-          end
-        end
+local function gen_rbl_callback(rule)
+  -- Here, we have functional approach: we form a pipeline of functions
+  -- f1, f2, ... fn. Each function accepts task and return boolean value
+  -- that allows to process pipeline further
+  -- Each function in the pipeline can add something to `dns_req` vector as a side effect
+
+  -- DNS requests to issue (might be hashed afterwards)
+  local dns_req = {}
+
+  local function add_dns_request(req, forced)
+    if dns_req[req] then
+      -- Duplicate request
+      if forced and not dns_req[req].forced then
+        dns_req[req].forced = true
       end
+    else
+      local nreq = {
+        forced = forced,
+        n = string.format('%s.%s',
+            maybe_make_hash(req, rule),
+            rule.rbl)
+      }
+      dns_req[req] = nreq
     end
   end
 
-  local params = {} -- indexed by rbl name
+  local function is_alive(_)
+    if rule.monitored then
+      if not rule.monitored:alive() then
+        return false
+      end
+    end
 
-  local function gen_rbl_rule(to_resolve, rbl)
-    lua_util.debugm(N, task, 'DNS REQUEST: label=%1 rbl=%2', to_resolve, rbl['symbol'])
-    if not params[to_resolve] then
-      local nrule = {
-        to_resolve = to_resolve,
-        rbls = {rbl},
-        forced = true,
-      }
-      nrule.callback = gen_rbl_callback(nrule)
-      params[to_resolve] = nrule
-    else
-      table.insert(params[to_resolve].rbls, rbl)
+    return true
+  end
+
+  local function check_user(task)
+    if task:get_user() then
+      return false
     end
 
-    return params[to_resolve]
+    return true
   end
 
-  local havegot = {
-    emails = {},
-    received = {},
-    dkim = {},
-  }
+  local function check_local(task)
+    local ip = task:get_from_ip()
 
-  local notgot = {}
+    if not ip:is_valid() then
+      ip = nil
+    end
 
-  local alive_rbls = fun.filter(function(_, rbl)
-    if rbl.monitored then
-      if not rbl.monitored:alive() then
-        return false
-      end
+    if ip and ip:is_local() or is_excluded_ip(ip) then
+      return false
     end
 
     return true
-  end, rbls)
-
-  -- Now exclude rbls, that are disabled by configuration
-  local enabled_rbls = fun.filter(function(_, rbl)
-    if rbl['exclude_users'] then
-      if not havegot['user'] and not notgot['user'] then
-        havegot['user'] = task:get_user()
-        if havegot['user'] == nil then
-          notgot['user'] = true
-        end
-      end
-      if havegot['user'] ~= nil then
-        return false
-      end
+  end
+
+  local function check_helo(task)
+    local helo = task:get_helo()
+
+    if not helo then
+      return false
     end
 
-    if (rbl['exclude_local'] or rbl['exclude_private_ips']) and not notgot['from'] then
-      if not havegot['from'] then
-        havegot['from'] = task:get_from_ip()
-        if not havegot['from']:is_valid() then
-          notgot['from'] = true
+    add_dns_request(helo, true)
+  end
+
+  local function check_dkim(task)
+    local das = task:get_symbol('DKIM_TRACE')
+    local mime_from_domain
+    local ret = false
+
+    if das and das[1] and das[1].options then
+
+      if rule.dkim_match_from then
+        -- We check merely mime from
+        mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain
+        if mime_from_domain then
+          mime_from_domain = rspamd_util.get_tld(mime_from_domain)
         end
       end
-      if havegot['from'] and not notgot['from'] and ((rbl['exclude_local'] and
-        is_excluded_ip(havegot['from'])) or (rbl['exclude_private_ips'] and
-        havegot['from']:is_local())) then
-        return false
-      end
-    end
 
-    -- Helo checks
-    if rbl['helo'] then
-      if notgot['helo'] then
-        return false
-      end
-      if not havegot['helo'] then
-        if rbl['hash'] then
-          havegot['helo'] = task:get_helo()
-          if havegot['helo'] then
-            havegot['helo'] = make_hash(havegot['helo'], rbl['hash'])
+      for _, d in ipairs(das[1].options) do
+
+        local domain,result = d:match('^([^%:]*):([%+%-%~])$')
+
+        -- We must ignore bad signatures, omg
+        if domain and result and result == '+' then
+          if rule.dkim_match_from then
+            -- We check merely mime from
+            local domain_tld = domain
+            if not rule.dkim_domainonly then
+              -- Adjust
+              domain_tld = rspamd_util.get_tld(domain)
+            end
+
+            if mime_from_domain and mime_from_domain == domain_tld then
+              add_dns_request(domain_tld, true)
+              ret = true
+            end
           else
-            notgot['helo'] = true
-            return false
-          end
-        else
-          havegot['helo'] = task:get_helo()
-          if havegot['helo'] == nil or not validate_dns(havegot['helo']) then
-            havegot['helo'] = nil
-            notgot['helo'] = true
-            return false
+            if rule.dkim_domainonly then
+              add_dns_request(rspamd_util.get_tld(domain), false)
+              ret = true
+            else
+              add_dns_request(domain, false)
+              ret = true
+            end
           end
         end
       end
-    elseif rbl['dkim'] then
-      -- DKIM checks
-      if notgot['dkim'] then
-        return false
-      end
-      if not havegot['dkim'] then
-        local das = task:get_symbol('DKIM_TRACE')
-        if ((das or E)[1] or E).options then
-          havegot['dkim'] = das[1]['options']
+    end
+
+    return ret
+  end
+
+  local function check_emails(task)
+    local emails = task:get_emails()
+
+    if not emails then
+      return false
+    end
+
+    for _,email in ipairs(emails) do
+      if rule.emails_domainonly then
+        dns_req[#dns_req + 1] = email:get_tld()
+      else
+        if rule.hash then
+          -- Leave @ as is
+          add_dns_request(string.format('%s@%s',
+              email:get_user(), email:get_domain()), false)
         else
-          notgot['dkim'] = true
-          return false
-        end
-      end
-    elseif rbl['emails'] then
-      -- Emails checks
-      if notgot['emails'] then
-        return false
-      end
-      if #havegot['emails'] == 0 then
-        havegot['emails'] = task:get_emails()
-        if havegot['emails'] == nil then
-          notgot['emails'] = true
-          havegot['emails'] = {}
-          return false
-        end
-      end
-    elseif rbl['from'] then
-      if notgot['from'] then
-        return false
-      end
-      if not havegot['from'] then
-        havegot['from'] = task:get_from_ip()
-        if not havegot['from']:is_valid() then
-          notgot['from'] = true
-          return false
-        end
-      end
-    elseif rbl['received'] then
-      if notgot['received'] then
-        return false
-      end
-      if #havegot['received'] == 0 then
-        havegot['received'] = task:get_received_headers()
-        if next(havegot['received']) == nil then
-          notgot['received'] = true
-          havegot['received'] = {}
-          return false
-        end
-      end
-    elseif rbl['rdns'] then
-      if notgot['rdns'] then
-        return false
-      end
-      if not havegot['rdns'] then
-        havegot['rdns'] = task:get_hostname()
-        if havegot['rdns'] == nil or havegot['rdns'] == 'unknown' then
-          notgot['rdns'] = true
-          return false
+          -- Replace @ with .
+          add_dns_request(string.format('%s.%s',
+              email:get_user(), email:get_domain()), false)
         end
       end
     end
 
     return true
-  end, alive_rbls)
-
-  -- Now we iterate over enabled rbls and fill params
-  -- Helo RBLs
-  fun.each(function(_, rbl)
-    local to_resolve = havegot['helo'] .. '.' .. rbl['rbl']
-    gen_rbl_rule(to_resolve, rbl)
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['helo'] then return true end
-    return false
-  end, enabled_rbls))
+  end
 
-  -- DKIM RBLs
-  fun.each(function(_, rbl)
-    local mime_from_domain
-    if rbl['dkim_match_from'] then
-      -- We check merely mime from
-      mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain
-      if mime_from_domain then
-        mime_from_domain = rspamd_util.get_tld(mime_from_domain)
-      end
-    end
+  local function check_from(task)
+    local ip = task:get_from_ip()
 
-    for _, d in ipairs(havegot['dkim']) do
-      local domain,result = d:match('^([^%:]*):([%+%-%~])$')
+    if not ip or not ip:is_valid() then
+      return true
+    end
+    if (ip:get_version() == 6 and rule.ipv6) or
+        (ip:get_version() == 4 and rule.ipv4) then
+      add_dns_request(ip_to_rbl(ip), true)
+    end
 
-      -- We must ignore bad signatures, omg
-      if domain and result and result == '+' then
+    return true
+  end
 
-        local to_resolve = domain .. '.' .. rbl['rbl']
+  local function check_received(task)
+    local received = fun.filter(function(h)
+      return not h['flags']['artificial']
+    end, task:get_received_headers()):totable()
 
-        if rbl['dkim_match_from'] then
-          -- We check merely mime from
-          local domain_tld = domain
-          if not rbl['dkim_domainonly'] then
-            -- Adjust
-            domain_tld = rspamd_util.get_tld(domain)
-          end
+    local received_total = #received
+    local check_conditions = gen_check_rcvd_conditions(rule, received_total)
 
-          if mime_from_domain and mime_from_domain == domain_tld then
-            gen_rbl_rule(to_resolve, rbl)
-          end
-        else
-          gen_rbl_rule(to_resolve, rbl)
-        end
+    for pos,rh in ipairs(received) do
+      if check_conditions(rh, pos) then
+        add_dns_request(ip_to_rbl(rh.real_ip), false)
       end
     end
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['dkim'] then return true end
-    return false
-  end, enabled_rbls))
-
-  -- Emails RBLs
-  fun.each(function(_, rbl)
-    if rbl['emails'] == 'domain_only' then
-      local cleanList = {}
-      for _, email in ipairs(havegot['emails']) do
-        cleanList[email:get_host()] = true
-      end
-      for k in pairs(cleanList) do
-        local to_resolve
-        if rbl['hash'] then
-          to_resolve = make_hash(tostring(k), rbl['hash']) .. '.' .. rbl['rbl']
-        else
-          to_resolve = k .. '.' .. rbl['rbl']
-        end
-        gen_rbl_rule(to_resolve, rbl)
-      end
-    else
-      for _, email in ipairs(havegot['emails']) do
-        local to_resolve
-        if rbl['hash'] then
-          to_resolve = make_hash(email:get_user() .. '@' .. email:get_host(), rbl['hash']) .. '.' .. rbl['rbl']
-        else
-          local upart = email:get_user()
-          if validate_dns(upart) then
-            to_resolve = upart .. '.' .. email:get_host() .. '.' .. rbl['rbl']
-          end
-        end
-        if to_resolve then
-          gen_rbl_rule(to_resolve, rbl)
-        end
-      end
+
+    return true
+  end
+
+  local function check_rdns(task)
+    local hostname = task:get_hostname()
+    if hostname == nil or hostname == 'unknown' then
+      return false
     end
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['emails'] then return true end
-    return false
-  end, enabled_rbls))
-
-  -- RDNS lists
-  fun.each(function(_, rbl)
-    local to_resolve = havegot['rdns'] .. '.' .. rbl['rbl']
-    gen_rbl_rule(to_resolve, rbl)
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['rdns'] then return true end
-    return false
-  end, enabled_rbls))
-
-  -- From lists
-  fun.each(function(_, rbl)
-    if (havegot['from']:get_version() == 6 and rbl['ipv6']) or
-      (havegot['from']:get_version() == 4 and rbl['ipv4']) then
-      local to_resolve = ip_to_rbl(havegot['from'], rbl['rbl'])
-      gen_rbl_rule(to_resolve, rbl)
-    end
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['from'] then return true end
-    return false
-  end, enabled_rbls))
 
-  havegot['received'] = fun.filter(function(h)
-    return not h['flags']['artificial']
-  end, havegot['received']):totable()
+    add_dns_request(hostname, true)
 
-  local received_total = #havegot['received']
-  -- Received lists
-  fun.each(function(_, rbl)
-    local check_conditions = gen_check_rcvd_conditions(rbl, received_total)
-    for pos,rh in ipairs(havegot['received']) do
-      if check_conditions(rh, pos) then
-        local to_resolve = ip_to_rbl(rh['real_ip'], rbl['rbl'])
-        local rule = gen_rbl_rule(to_resolve, rbl)
-        -- Disable forced for received resolving, as we have no control on
-        -- those headers count
-        rule.forced = false
+    return true
+  end
+
+  -- Create function pipeline depending on rbl settings
+  local pipeline = {
+    is_alive, -- generic for all
+  }
+
+  if rule.exclude_users then
+    pipeline[#pipeline + 1] = check_user
+  end
+
+  if rule.exclude_local or rule.exclude_private_ips then
+    pipeline[#pipeline + 1] = check_local
+  end
+
+  if rule.helo then
+    pipeline[#pipeline + 1] = check_helo
+  end
+
+  if rule.dkim then
+    pipeline[#pipeline + 1] = check_dkim
+  end
+
+  if rule.emails then
+    pipeline[#pipeline + 1] = check_emails
+  end
+
+  if rule.from then
+    pipeline[#pipeline + 1] = check_from
+  end
+
+  if rule.received then
+    pipeline[#pipeline + 1] = check_received
+  end
+
+  if rule.rdns then
+    pipeline[#pipeline + 1] = check_rdns
+  end
+
+  return function(task)
+    local function rbl_dns_callback(_, to_resolve, results, err)
+      rbl_dns_process(task, rule, to_resolve, results, err)
+    end
+
+    -- Execute functions pipeline
+    for _,f in ipairs(pipeline) do
+      if not f(task) then
+        lua_util.debugm(N, task, "skip rbl check: %s; pipeline condition returned false",
+            rule.symbol)
+        return
       end
     end
-  end,
-  fun.filter(function(_, rbl)
-    if rbl['received'] then return true end
-    return false
-  end, enabled_rbls))
 
-  local r = task:get_resolver()
-  for _,p in pairs(params) do
-    r:resolve_a({
-      task = task,
-      name = p.to_resolve,
-      callback = p.callback,
-      forced = p.forced
-    })
+    -- Now check all DNS requests pending and emit them
+    local r = task:get_resolver()
+    for name,p in ipairs(dns_req) do
+      if validate_dns(p.n) then
+        lua_util.debugm(N, task, "rbl %s; resolve %s -> %s",
+            rule.symbol, name, p.n)
+        r:resolve_a({
+          task = task,
+          name = p.n,
+          callback = rbl_dns_callback,
+          forced = p.forced
+        })
+      else
+        rspamd_logger.warnx(task, 'cannot send invalid DNS request %s for %s',
+            p.n, rule.symbol)
+      end
+    end
   end
 end
 
@@ -514,19 +472,21 @@ local default_defaults = {
   ['default_ipv4'] = true,
   ['default_ipv6'] = true,
   ['default_received'] = false,
-  ['default_from'] = false,
+  ['default_from'] = true,
   ['default_unknown'] = false,
   ['default_rdns'] = false,
   ['default_helo'] = false,
   ['default_dkim'] = false,
   ['default_dkim_domainonly'] = true,
   ['default_emails'] = false,
+  ['default_emails_domainonly'] = false,
   ['default_exclude_private_ips'] = true,
   ['default_exclude_users'] = false,
   ['default_exclude_local'] = true,
   ['default_is_whitelist'] = false,
   ['default_ignore_whitelist'] = false,
 }
+-- Enrich with defaults
 for default, default_v in pairs(default_defaults) do
   if opts[default] == nil then
     opts[default] = default_v
@@ -540,140 +500,175 @@ end
 
 local white_symbols = {}
 local black_symbols = {}
-local need_dkim = false
 
-local id = rspamd_config:register_symbol({
-  type = 'callback',
-  callback = rbl_cb,
-  name = 'RBL_CALLBACK',
-  flags = 'empty,nice'
+local rule_schema = ts.shape({
+  enabled = ts.boolean:is_optional(),
+  disabled = ts.boolean:is_optional(),
+  rbl = ts.string,
+  symbol = ts.string:is_optional(),
+  returncodes = ts.map_of(
+      ts.string / string.upper,
+      (
+          ts.array_of(ts.string) + (ts.string / function(s)
+            return { s }
+          end)
+      )
+  ):is_optional(),
+  whitelist_exception = (
+      ts.array_of(ts.string) + (ts.string / function(s) return {s} end)
+  ):is_optional(),
+  local_exclude_ip_map = ts.string:is_optional(),
+  hash = ts.one_of{"sha1", "sha256", "sha384", "sha512", "md5", "blake2"}:is_optional(),
+  hash_format = ts.one_of{"hex", "base32", "base64"}:is_optional(),
+  hash_len = (ts.integer + ts.string / tonumber):is_optional(),
+}, {
+  extra_fields = ts.map_of(ts.string, ts.boolean)
 })
 
-local is_monitored = {}
-local rbls_count = 0
-for key,rbl in pairs(opts['rbls']) do
-  (function()
-    if type(rbl) ~= 'table' or rbl['disabled'] then
-      rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key)
-      return
-    end
 
-    for default,_ in pairs(default_defaults) do
-      local rbl_opt = default:gsub('^default_', '')
-      if rbl[rbl_opt] == nil then
-        rbl[rbl_opt] = opts[default]
-      end
-    end
+local monitored_addresses = {}
 
-    if not rbl['enabled'] then return end
+local function add_rbl(key, rbl)
+  if not rbl.symbol then
+    rbl.symbol = key:upper()
+  end
 
-    if type(rbl['returncodes']) == 'table' then
-      for s,_ in pairs(rbl['returncodes']) do
-        if type(rspamd_config.get_api_version) ~= 'nil' then
-          rspamd_config:register_symbol({
-            name = s,
-            parent = id,
-            type = 'virtual'
-          })
+  local flags_tbl = {}
+  if rbl.is_whitelist then
+    flags_tbl[#flags_tbl + 1] = 'nice'
+  end
 
-          if rbl['dkim'] then
-            need_dkim = true
-          end
-          if(rbl['is_whitelist']) then
-            if type(rbl['whitelist_exception']) == 'string' then
-              if (rbl['whitelist_exception'] ~= s) then
-                table.insert(white_symbols, s)
-              end
-            elseif type(rbl['whitelist_exception']) == 'table' then
-              local foundException = false
-              for _, e in pairs(rbl['whitelist_exception']) do
-                if e == s then
-                  foundException = true
-                  break
-                end
-              end
-              if not foundException then
-                table.insert(white_symbols, s)
-              end
-            else
-                table.insert(white_symbols, s)
-            end
-          else
-            if rbl['ignore_whitelists'] == false then
-              table.insert(black_symbols, s)
-            end
-          end
-        end
-      end
-    end
-    if not rbl['symbol'] and
-      ((rbl['returncodes'] and rbl['unknown']) or
-      (not rbl['returncodes'])) then
-        rbl['symbol'] = key
-    end
-    if rbl['symbol'] then
+  if not (rbl.dkim or rbl.emails) then
+    flags_tbl[#flags_tbl + 1] = 'empty'
+  end
+
+  local id = rspamd_config:register_symbol{
+    type = 'callback',
+    callback = gen_rbl_callback(rbl),
+    name = rbl.symbol,
+    flags = table.concat(flags_tbl, ',')
+  }
+
+  if rbl.dkim then
+    rspamd_config:register_dependency(rbl.symbol, 'DKIM_CHECK')
+  end
+
+  if rbl.returncodes then
+    for s,_ in pairs(rbl['returncodes']) do
       rspamd_config:register_symbol({
-        name = rbl['symbol'],
+        name = s,
         parent = id,
         type = 'virtual'
       })
-      rbls_count = rbls_count + 1
 
-      if rbl['dkim'] then
-        need_dkim = true
-      end
-      if (rbl['is_whitelist']) then
-            if type(rbl['whitelist_exception']) == 'string' then
-              if (rbl['whitelist_exception'] ~= rbl['symbol']) then
-                table.insert(white_symbols, rbl['symbol'])
-              end
-            elseif type(rbl['whitelist_exception']) == 'table' then
-              local foundException = false
-              for _, e in pairs(rbl['whitelist_exception']) do
-                if e == rbl['symbol'] then
-                  foundException = true
-                  break
-                end
-              end
-              if not foundException then
-                table.insert(white_symbols, rbl['symbol'])
-              end
-            else
-              table.insert(white_symbols, rbl['symbol'])
+      if rbl.is_whitelist then
+        if rbl.whitelist_exception then
+          local foundException = false
+          for _, e in ipairs(rbl.whitelist_exception) do
+            if e == s then
+              foundException = true
+              break
             end
+          end
+          if not foundException then
+            table.insert(white_symbols, s)
+          end
+        else
+          table.insert(white_symbols, s)
+        end
       else
-        if rbl['ignore_whitelists'] == false then
-          table.insert(black_symbols, rbl['symbol'])
+        if rbl.ignore_whitelists == false then
+          table.insert(black_symbols, s)
         end
       end
     end
-    if rbl['rbl'] then
-      if not rbl['disable_monitoring'] and not rbl['is_whitelist'] and
-          not is_monitored[rbl['rbl']] then
-        is_monitored[rbl['rbl']] = true
-        rbl.monitored = rspamd_config:register_monitored(rbl['rbl'], 'dns',
+  end
+
+  if not rbl.is_whitelist and rbl.ignore_whitelists == false then
+    table.insert(black_symbols, rbl.symbol)
+  end
+  -- Process monitored
+  if not rbl.disable_monitoring and not rbl.is_whitelist then
+    if not monitored_addresses[rbl.rbl] then
+      monitored_addresses[rbl.rbl] = true
+      rbl.monitored = rspamd_config:register_monitored(rbl['rbl'], 'dns',
           {
             rcode = 'nxdomain',
-            prefix = rbl['monitored_address'] or default_monitored
+            prefix = rbl.monitored_address or default_monitored
           })
+    end
+  end
+end
+
+for key,rbl in pairs(opts['rbls']) do
+  if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then
+    rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key)
+  else
+    for default,_ in pairs(default_defaults) do
+      local rbl_opt = default:sub(#('default_') + 1)
+      if rbl[rbl_opt] == nil then
+        rbl[rbl_opt] = opts[default]
       end
+    end
 
-      rbls[key] = rbl
+    local res,err = rule_schema:transform(rbl)
+    if not res then
+      rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
+          key, err)
+    else
+      add_rbl(key, res)
     end
-  end)()
+  end -- rbl.enabled
 end
 
-if rbls_count == 0 then
-  lua_util.disable_module(N, "config")
-end
+-- We now create two symbols:
+-- * RBL_CALLBACK_WHITE that depends on all symbols white
+-- * RBL_CALLBACK that depends on all symbols black to participate in depends chains
+
+local function rbl_callback_white(task)
+  local found_whitelist = false
+  for _, w in ipairs(white_symbols) do
+    if task:has_symbol(w) then
+      lua_util.debugm(N, task,'found whitelist %s', w)
+      found_whitelist = true
+      break
+    end
+  end
 
-for _, w in pairs(white_symbols) do
-  for _, b in pairs(black_symbols) do
-    local csymbol = 'RBL_COMPOSITE_' .. w .. '_' .. b
-    rspamd_config:set_metric_symbol(csymbol, 0, 'Autogenerated composite')
-    rspamd_config:add_composite(csymbol, w .. ' & ' .. b)
+  if found_whitelist then
+    -- Disable all symbols black
+    for _, b in ipairs(black_symbols) do
+      lua_util.debugm(N, task,'disable %s, whitelist found', b)
+      task:disable_symbol(b)
+    end
   end
+  lua_util.debugm(N, task, "finished rbl whitelists processing")
+end
+
+local function rbl_callback_fin(task)
+  -- Do nothing
+  lua_util.debugm(N, task, "finished rbl processing")
+end
+
+rspamd_config:register_symbol{
+  type = 'callback',
+  callback = rbl_callback_white,
+  name = 'RBL_CALLBACK_WHITE',
+  flags = 'nice,empty'
+}
+
+rspamd_config:register_symbol{
+  type = 'callback',
+  callback = rbl_callback_fin,
+  name = 'RBL_CALLBACK',
+  flags = 'empty'
+}
+
+for _, w in ipairs(white_symbols) do
+  rspamd_config:register_dependency('RBL_CALLBACK_WHITE', w)
 end
-if need_dkim then
-  rspamd_config:register_dependency('RBL_CALLBACK', 'DKIM_CHECK')
+
+for _, b in ipairs(black_symbols) do
+  rspamd_config:register_dependency(b, 'RBL_CALLBACK_WHITE')
+  rspamd_config:register_dependency('RBL_CALLBACK', b)
 end
-- 
2.39.5