mirrors
/
rspamd
mirror of https://github.com/vstakhov/rspamd.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 159 Wiki Activity
Browse Source

Merge pull request #3960 from HeinleinSupport/lua_scanners/icap 

lua_scanners/icap
tags/3.2
Vsevolod Stakhov 2 years ago
parent
b248bd715a 78ca6debc2
commit
bcf3daa54b
No account linked to committer's email address
1 changed files with 345 additions and 86 deletions
Split View Show Diff Stats
  1. 345
    86
      lualib/lua_scanners/icap.lua

+ 345
- 86
lualib/lua_scanners/icap.lua View File

@@ -15,17 +15,69 @@ See the License for the specific language governing permissions and
limitations under the License.
]]--

--[[[
--]]
--[[
@module icap
This module contains icap access functions.
Currently tested with
- C-ICAP Squidclamav / echo
- F-Secure Internet Gatekeeper
- Kaspersky Web Traffic Security
- Kaspersky Scan Engine 2.0
- McAfee Web Gateway 11
- Sophos Savdi
- Symantec (Rspamd <3.2, >=3.2 untested)
- Trend Micro IWSVA 6.0
- Trend Micro Web Gateway

@TODO
- Preview / Continue
- Reqmod URL's
- Content-Type / Filename
]] --

--[[
Configuration Notes:

C-ICAP Squidclamav
scheme = "squidclamav";

ESET Gateway Security / Antivirus for Linux example:
scheme = "scan";

F-Secure Internet Gatekeeper example:
scheme = "respmod";
x_client_header = true;
x_rcpt_header = true;
x_from_header = true;

Kaspersky Web Traffic Security example:
scheme = "av/respmod";
x_client_header = true;

Kaspersky Web Traffic Security (as configured in kavicapd.xml):
scheme = "resp";
x_client_header = true;

McAfee Web Gateway 11 (Headers must be activated with personal extra Rules)
scheme = "respmod";
x_client_header = true;

Sophos SAVDI example:
# scheme as configured in savdi.conf (name option in service section)
scheme = "respmod";

Symantec example:
scheme = "avscan";

Trend Micro IWSVA example (X-Virus-ID/X-Infection-Found headers must be activated):
scheme = "avscan";
x_client_header = true;

Trend Micro Web Gateway example (X-Virus-ID/X-Infection-Found headers must be activated):
scheme = "interscan";
x_client_header = true;
]] --


local lua_util = require "lua_util"
local tcp = require "rspamd_tcp"
@@ -48,6 +100,8 @@ local function icap_config(opts)
scan_image_mime = false,
scheme = "scan",
default_port = 1344,
ssl = false,
no_ssl_verify = false,
timeout = 10.0,
log_clean = false,
retransmits = 2,
@@ -58,6 +112,14 @@ local function icap_config(opts)
action = false,
dynamic_scan = false,
user_agent = "Rspamd",
x_client_header = false,
x_rcpt_header = false,
x_from_header = false,
req_headers_enabled = true,
req_fake_url = "http://127.0.0.1/mail",
http_headers_enabled = true,
use_http_result_header = true,
use_http_3xx_as_threat = false,
}

icap_conf = lua_util.override_defaults(icap_conf, opts)
@@ -103,7 +165,9 @@ local function icap_check(task, content, digest, rule, maybe_part)
local upstream = rule.upstreams:get_upstream_round_robin()
local addr = upstream:get_addr()
local retransmits = rule.retransmits
local respond_headers = {}
local http_headers = {}
local req_headers = {}
local tcp_options = {}

-- Build extended User Agent
if rule.user_agent == "extended" then
@@ -116,16 +180,24 @@ local function icap_check(task, content, digest, rule, maybe_part)

-- Build the icap queries
local options_request = {
string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(true), rule.scheme),
string.format("OPTIONS icap://%s/%s ICAP/1.0\r\n", addr:to_string(), rule.scheme),
string.format('Host: %s\r\n', addr:to_string()),
string.format("User-Agent: %s\r\n", rule.user_agent),
"Connection: keep-alive\r\n",
"Encapsulated: null-body=0\r\n\r\n",
}
if rule.user_agent == "none" then
table.remove(options_request, 3)
end

local size = string.format("%x", tonumber(#content))
local respond_headers = {
-- Add main RESPMOD header before any other
string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme),
string.format('Host: %s\r\n', addr:to_string()),
}

local size = tonumber(#content)
local chunked_size = string.format("%x", size)

local function icap_callback(err, conn)

@@ -149,16 +221,13 @@ local function icap_check(task, content, digest, rule, maybe_part)
lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
rule.log_prefix, addr, addr:get_port())

tcp.request({
task = task,
host = addr:to_string(),
port = addr:get_port(),
timeout = rule.timeout,
stop_pattern = '\r\n',
data = options_request,
read = false,
callback = icap_callback,
})
tcp_options.host = addr:to_string()
tcp_options.port = addr:get_port()
tcp_options.callback = icap_callback
tcp_options.data = options_request

tcp.request(tcp_options)

else
rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
'exceed - error: %s', rule.log_prefix, err_m or '')
@@ -167,11 +236,76 @@ local function icap_check(task, content, digest, rule, maybe_part)
end
end

local function get_req_headers()

local req_hlen = 2
table.insert(req_headers, string.format('GET %s HTTP/1.0\r\n', rule.req_fake_url))
table.insert(req_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
--table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html'))
if rule.user_agent ~= "none" then
table.insert(req_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
end

for _, h in ipairs(req_headers) do
req_hlen = req_hlen + tonumber(#h)
end

return req_hlen, req_headers

end

local function get_http_headers()
local http_hlen = 2
table.insert(http_headers, 'HTTP/1.0 200 OK\r\n')
table.insert(http_headers, string.format('Date: %s\r\n', rspamd_util.time_to_string(rspamd_util.get_time())))
table.insert(http_headers, string.format('Server: %s\r\n', 'Apache/2.4'))
if rule.user_agent ~= "none" then
table.insert(http_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
end
--table.insert(http_headers, string.format('Content-Type: %s\r\n', 'text/html'))
table.insert(http_headers, string.format('Content-Length: %s\r\n', size))

for _, h in ipairs(http_headers) do
http_hlen = http_hlen + tonumber(#h)
end

return http_hlen, http_headers

end

local function get_respond_query()
table.insert(respond_headers, 1, string.format(
'RESPMOD icap://%s:%s/%s ICAP/1.0\r\n', addr:to_string(), addr:get_port(), rule.scheme))
local req_hlen = 0
local resp_req_headers
local http_hlen = 0
local resp_http_headers

-- Append all extra headers
if rule.user_agent ~= "none" then
table.insert(respond_headers, string.format("User-Agent: %s\r\n", rule.user_agent))
end

if rule.req_headers_enabled then
req_hlen, resp_req_headers = get_req_headers()
end
if rule.http_headers_enabled then
http_hlen, resp_http_headers = get_http_headers()
end

if rule.req_headers_enabled and rule.http_headers_enabled then
local res_body_hlen = req_hlen + http_hlen
table.insert(respond_headers, string.format('Encapsulated: req-hdr=0, res-hdr=%s, res-body=%s\r\n', req_hlen, res_body_hlen))
elseif rule.http_headers_enabled then
table.insert(respond_headers, string.format('Encapsulated: res-hdr=0, res-body=%s\r\n', http_hlen))
else
table.insert(respond_headers, 'Encapsulated: res-body=0\r\n')
end

table.insert(respond_headers, '\r\n')
for _,h in ipairs(resp_req_headers) do table.insert(respond_headers, h) end
table.insert(respond_headers, '\r\n')
table.insert(respond_headers, size .. '\r\n')
for _,h in ipairs(resp_http_headers) do table.insert(respond_headers, h) end
table.insert(respond_headers, '\r\n')
table.insert(respond_headers, chunked_size .. '\r\n')
table.insert(respond_headers, content)
table.insert(respond_headers, '\r\n0\r\n\r\n')
return respond_headers
@@ -183,16 +317,17 @@ local function icap_check(task, content, digest, rule, maybe_part)
end
end

local function icap_result_header_table(result)
local function result_header_table(result)
local icap_headers = {}
for s in result:gmatch("[^\r\n]+") do
if string.find(s, '^ICAP') then
icap_headers['icap'] = s
end
if string.find(s, '[%a%d-+]-:') then
icap_headers['icap'] = tostring(s)
elseif string.find(s, '^HTTP') then
icap_headers['http'] = tostring(s)
elseif string.find(s, '[%a%d-+]-:') then
local _,_,key,value = tostring(s):find("([%a%d-+]-):%s?(.+)")
if key ~= nil then
icap_headers[key] = value
icap_headers[key:lower()] = tostring(value)
end
end
end
@@ -201,7 +336,7 @@ local function icap_check(task, content, digest, rule, maybe_part)
return icap_headers
end

local function icap_parse_result(icap_headers)
local function icap_parse_result(headers)

local threat_string = {}

@@ -209,41 +344,65 @@ local function icap_check(task, content, digest, rule, maybe_part)
@ToDo: handle type in response

Generic Strings:
X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;
icap: X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
icap: X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;

Symantec String:
X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;
icap: X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
icap: X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;

Sophos Strings:
X-Virus-ID: Troj/DocDl-OYC
icap: X-Virus-ID: Troj/DocDl-OYC
http: X-Blocked: Virus found during virus scan
http: X-Blocked-By: Sophos Anti-Virus

Kaspersky Web Traffic Security Strings:
X-Virus-ID: HEUR:Backdoor.Java.QRat.gen
X-Response-Info: blocked
X-Virus-ID: no threats
X-Response-Info: blocked
X-Response-Info: passed

Trend Micro IWSVA Strings:
X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1
X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1;
icap: X-Virus-ID: HEUR:Backdoor.Java.QRat.gen
icap: X-Response-Info: blocked
icap: X-Virus-ID: no threats
icap: X-Response-Info: blocked
icap: X-Response-Info: passed
http: HTTP/1.1 403 Forbidden

Kaspersky Scan Engine 2.0 (ICAP mode)
icap: X-Virus-ID: EICAR-Test-File
http: HTTP/1.0 403 Forbidden

Trend Micro Strings:
icap: X-Virus-ID: Trojan.W97M.POWLOAD.SMTHF1
icap: X-Infection-Found: Type=0; Resolution=2; Threat=Trojan.W97M.POWLOAD.SMTHF1;
http: HTTP/1.1 403 Forbidden (TMWS Blocked)
http: HTTP/1.1 403 Forbidden

F-Secure Internet Gatekeeper Strings:
X-FSecure-Scan-Result: infected
X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203"
X-FSecure-Infected-Filename: "virus.doc"
icap: X-FSecure-Scan-Result: infected
icap: X-FSecure-Infection-Name: "Malware.W97M/Agent.32584203"
icap: X-FSecure-Infected-Filename: "virus.doc"

ESET File Security for Linux 7.0
X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA;
X-Virus-ID: Trojaner
X-Response-Info: Blocked
icap: X-Infection-Found: Type=0; Resolution=0; Threat=VBA/TrojanDownloader.Agent.JOA;
icap: X-Virus-ID: Trojaner
icap: X-Response-Info: Blocked

McAfee Web Gateway 11 (Headers must be activated with personal extra Rules)
icap: X-Virus-ID: EICAR test file
icap: X-Media-Type: text/plain
icap: X-Block-Result: 80
icap: X-Block-Reason: Malware found
icap: X-Block-Reason: Archive not supported
icap: X-Block-Reason: Media Type (Block List)
http: HTTP/1.0 403 VirusFound

C-ICAP Squidclamav
icap/http: X-Infection-Found: Type=0; Resolution=2; Threat={HEX}EICAR.TEST.3.UNOFFICIAL;
icap/http: X-Virus-ID: {HEX}EICAR.TEST.3.UNOFFICIAL
http: HTTP/1.0 307 Temporary Redirect
]] --

if icap_headers['X-Infection-Found'] then
-- Generic ICAP Headers
if headers['x-infection-found'] then
local _,_,icap_type,_,icap_threat =
icap_headers['X-Infection-Found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")

if not icap_type or icap_type == 2 then
-- error returned
@@ -251,35 +410,37 @@ local function icap_check(task, content, digest, rule, maybe_part)
'%s: icap error X-Infection-Found: %s', rule.log_prefix, icap_threat)
common.yield_result(task, rule, icap_threat, 0,
'fail', maybe_part)
return true
else
lua_util.debugm(rule.name, task,
'%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat)
table.insert(threat_string, icap_threat)
end

elseif icap_headers['X-Virus-ID'] and icap_headers['X-Virus-ID'] ~= "no threats" then
elseif headers['x-virus-id'] and headers['x-virus-id'] ~= "no threats" then
lua_util.debugm(rule.name, task,
'%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID'])
'%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id'])

if string.find(icap_headers['X-Virus-ID'], ', ') then
local vnames = lua_util.str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {}
if string.find(headers['x-virus-id'], ', ') then
local vnames = lua_util.str_split(string.gsub(headers['x-virus-id'], "%s", ""), ',') or {}

for _,v in ipairs(vnames) do
table.insert(threat_string, v)
end
else
table.insert(threat_string, icap_headers['X-Virus-ID'])
table.insert(threat_string, headers['x-virus-id'])
end
elseif icap_headers['X-FSecure-Scan-Result'] and icap_headers['X-FSecure-Scan-Result'] ~= "clean" then
-- FSecure X-Headers
elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then

local infected_filename = ""
local infection_name = "-unknown-"

if icap_headers['X-FSecure-Infected-Filename'] then
infected_filename = string.gsub(icap_headers['X-FSecure-Infected-Filename'], '[%s"]', '')
if headers['x-fsecure-infected-filename'] then
infected_filename = string.gsub(headers['x-fsecure-infected-filename'], '[%s"]', '')
end
if icap_headers['X-FSecure-Infection-Name'] then
infection_name = string.gsub(icap_headers['X-FSecure-Infection-Name'], '[%s"]', '')
if headers['x-fsecure-infection-name'] then
infection_name = string.gsub(headers['x-fsecure-infection-name'], '[%s"]', '')
end

lua_util.debugm(rule.name, task,
@@ -295,13 +456,54 @@ local function icap_check(task, content, digest, rule, maybe_part)
else
table.insert(threat_string, infection_name)
end
-- McAfee Web Gateway manual extra headers
elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then
table.insert(threat_string, headers['x-mwg-block-reason'])
-- Sophos SAVDI special http headers
elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then
table.insert(threat_string, headers['x-blocked'])
-- last try HTTP [4]xx return
elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then
local message = string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', ''))
table.insert(threat_string, message)
elseif rule.use_http_3xx_as_threat and headers.http and string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d') then
local message = string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', ''))
table.insert(threat_string, message)
end

if #threat_string > 0 then
common.yield_result(task, rule, threat_string, rule.default_score, nil, maybe_part)
common.save_cache(task, digest, rule, threat_string, rule.default_score, maybe_part)
return true
else
common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
common.log_clean(task, rule)
return false
end
end

local function icap_r_respond_http_cb(err_m, data, connection)
if err_m or connection == nil then
icap_requery(err_m, "icap_r_respond_http_cb")
else
local result = tostring(data)

local icap_http_headers = result_header_table(result) or {}
-- Find HTTP/[12].x [234]xx response
if icap_http_headers.http and string.find(icap_http_headers.http, 'HTTP%/[12]%.. [234]%d%d') then
local icap_http_header_result = icap_parse_result(icap_http_headers)
if icap_http_header_result then
-- Threat found - close connection
connection:close()
else
common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
common.log_clean(task, rule)
end
else
rspamd_logger.errx(task, '%s: unhandled response |%s|',
rule.log_prefix, string.gsub(result, "\r\n", ", "))
common.yield_result(task, rule,
'unhandled icap response: ' .. icap_http_headers.icap or "-",
0.0, 'fail', maybe_part)
end
end
end

@@ -310,12 +512,28 @@ local function icap_check(task, content, digest, rule, maybe_part)
icap_requery(err_m, "icap_r_respond_cb")
else
local result = tostring(data)
conn:close()

local icap_headers = icap_result_header_table(result) or {}
local icap_headers = result_header_table(result) or {}
-- Find ICAP/1.x 2xx response
if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
icap_parse_result(icap_headers)
local icap_header_result = icap_parse_result(icap_headers)
if icap_header_result then
-- Threat found - close connection
connection:close()
elseif not icap_header_result
and rule.use_http_result_header
and icap_headers.encapsulated
and not string.find(icap_headers.encapsulated, 'null%-body=0')
then
-- Try to read encapsulated HTTP Headers
lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers',
rule.log_prefix)
connection:add_read(icap_r_respond_http_cb, '\r\n\r\n')
else
connection:close()
common.save_cache(task, digest, rule, 'OK', 0, maybe_part)
common.log_clean(task, rule)
end
elseif icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then
-- Find ICAP/1.x 5/4xx response
--[[
@@ -323,6 +541,12 @@ local function icap_check(task, content, digest, rule, maybe_part)
ICAP/1.0 539 Aborted - No AV scanning license
SquidClamAV/C-ICAP:
ICAP/1.0 500 Server error
Eset:
ICAP/1.0 405 Forbidden
TrendMicro:
ICAP/1.0 400 Bad request
McAfee:
ICAP/1.0 418 Bad composition
]]--
rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
common.yield_result(task, rule, icap_headers.icap, 0.0,
@@ -350,32 +574,50 @@ local function icap_check(task, content, digest, rule, maybe_part)
if err_m or connection == nil then
icap_requery(err_m, "icap_r_options_cb")
else
local icap_headers = icap_result_header_table(tostring(data))
local icap_headers = result_header_table(tostring(data))

if icap_headers.icap and string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
if icap_headers['Methods'] and string.find(icap_headers['Methods'], 'RESPMOD') then
if icap_headers['Allow'] and string.find(icap_headers['Allow'], '204') then
if icap_headers['methods'] and string.find(icap_headers['methods'], 'RESPMOD') then
-- Allow "204 No Content" responses
-- https://datatracker.ietf.org/doc/html/rfc3507#section-4.6
if icap_headers['allow'] and string.find(icap_headers['allow'], '204') then
add_respond_header('Allow', '204')
end
if icap_headers['Service'] and string.find(icap_headers['Service'], 'IWSVA 6.5') then
add_respond_header('Encapsulated', 'res-hdr=0 res-body=0')
else
add_respond_header('Encapsulated', 'res-body=0')
end
if icap_headers['Server'] and string.find(icap_headers['Server'], 'F-Secure ICAP Server') then
local from = task:get_from('mime')
local rcpt_to = task:get_principal_recipient()

if rule.x_client_header then
local client = task:get_from_ip()
if client then add_respond_header('X-Client-IP', client:to_string()) end
add_respond_header('X-Mail-From', from[1].addr)
add_respond_header('X-Rcpt-To', rcpt_to)
end

conn:add_write(icap_w_respond_cb, get_respond_query())
-- F-Secure extra headers
if icap_headers['server'] and string.find(icap_headers['server'], 'f-secure icap server') then

if rule.x_rcpt_header then
local rcpt_to = task:get_principal_recipient()
if rcpt_to then add_respond_header('X-Rcpt-To', rcpt_to) end
end

if rule.x_from_header then
local mail_from = task:get_principal_recipient()
if mail_from and mail_from[1] then add_respond_header('X-Rcpt-To', mail_from[1].addr) end
end

end

if icap_headers.connection and icap_headers.connection:lower() == 'close' then
lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection',
rule.log_prefix, icap_headers.connection)
connection:close()
tcp_options.callback = icap_w_respond_cb
tcp_options.data = get_respond_query()
tcp.request(tcp_options)
else
connection:add_write(icap_w_respond_cb, get_respond_query())
end

else
rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
rule.log_prefix, icap_headers['Methods'])
rule.log_prefix, icap_headers['methods'])
common.yield_result(task, rule, 'NO RESPMOD', 0.0,
'fail', maybe_part)
end
@@ -397,16 +639,24 @@ local function icap_check(task, content, digest, rule, maybe_part)
end
end

tcp.request({
task = task,
host = addr:to_string(),
port = addr:get_port(),
timeout = rule.timeout,
stop_pattern = '\r\n',
data = options_request,
read = false,
callback = icap_callback,
})
tcp_options.task = task
tcp_options.stop_pattern = '\r\n'
tcp_options.read = false
tcp_options.timeout = rule.timeout
tcp_options.callback = icap_callback
tcp_options.data = options_request

if rule.ssl then
tcp_options.ssl = true
if rule.no_ssl_verify then
tcp_options.no_ssl_verify = true
end
end

tcp_options.host = addr:to_string()
tcp_options.port = addr:get_port()

tcp.request(tcp_options)
end

if common.condition_check_and_continue(task, content, rule, digest,

Write Preview
Loading…
Cancel
Save