1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
--[[
Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--
--[[[
-- @module virustotal
-- This module contains Virustotal integaration support
-- https://www.virustotal.com/
--]]
local lua_util = require "lua_util"
local http = require "rspamd_http"
local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
local N = 'virustotal'
local function virustotal_config(opts)
local default_conf = {
name = N,
url = 'https://www.virustotal.com/vtapi/v2/file',
timeout = 5.0,
log_clean = false,
retransmits = 1,
cache_expire = 7200, -- expire redis in 2h
message = '${SCANNER}: spam message found: "${VIRUS}"',
detection_category = "virus",
default_score = 1,
action = false,
scan_mime_parts = true,
scan_text_mime = false,
scan_image_mime = false,
apikey = nil, -- Required to set by user
-- Specific for virustotal
minimum_engines = 3, -- Minimum required to get scored
full_score_engines = 7, -- After this number we set max score
}
default_conf = lua_util.override_defaults(default_conf, opts)
if not default_conf.prefix then
default_conf.prefix = 'rs_' .. default_conf.name .. '_'
end
if not default_conf.log_prefix then
if default_conf.name:lower() == default_conf.type:lower() then
default_conf.log_prefix = default_conf.name
else
default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
end
end
if not default_conf.apikey then
rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks')
return nil
end
lua_util.add_debug_alias('external_services', default_conf.name)
return default_conf
end
local function virustotal_check(task, content, digest, rule, maybe_part)
local function virustotal_check_uncached()
local function make_url(hash)
return string.format('%s/report?apikey=%s&resource=%s',
rule.url, rule.apikey, hash)
end
local hash = rspamd_cryptobox_hash.create_specific('md5')
hash:update(content)
hash = hash:hex()
local url = make_url(hash)
lua_util.debugm(N, task, "send request %s", url)
local request_data = {
task = task,
url = url,
timeout = rule.timeout,
}
local function vt_http_callback(http_err, code, body, headers)
if http_err then
rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers)
else
local cached
local dyn_score
-- Parse the response
if code ~= 200 then
if code == 404 then
cached = 'OK'
if rule['log_clean'] then
rspamd_logger.infox(task, '%s: hash %s clean (not found)',
rule.log_prefix, hash)
else
lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
rule.log_prefix, hash)
end
elseif code == 204 then
-- Request rate limit exceeded
rspamd_logger.infox(task, 'virustotal request rate limit exceeded')
task:insert_result(rule.symbol_fail, 1.0, 'rate limit exceeded')
return
else
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
return
end
else
local ucl = require "ucl"
local parser = ucl.parser()
local res,json_err = parser:parse_string(body)
lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
rule.log_prefix, body)
if res then
local obj = parser:get_object()
if not obj.positives or type(obj.positives) ~= 'number' then
if obj.response_code then
if obj.response_code == 0 then
cached = 'OK'
if rule['log_clean'] then
rspamd_logger.infox(task, '%s: hash %s clean (not found)',
rule.log_prefix, hash)
else
lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
rule.log_prefix, hash)
end
else
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
'bad response code: ' .. tostring(obj.response_code), body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
return
end
else
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
'no response_code', body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
return
end
else
if obj.positives < rule.minimum_engines then
lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min',
rule.log_prefix, obj.positives, rule.minimum_engines)
-- TODO: add proper hashing!
cached = 'OK'
else
if obj.positives > rule.full_score_engines then
dyn_score = 1.0
else
local norm_pos = obj.positives - rule.minimum_engines
dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines)
end
if dyn_score < 0 or dyn_score > 1 then
dyn_score = 1.0
end
local sopt = string.format("%s:%s/%s",
hash, obj.positives, obj.total)
common.yield_result(task, rule, sopt, dyn_score, nil, maybe_part)
cached = sopt
end
end
else
-- not res
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
json_err, body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err)
return
end
end
if cached then
common.save_cache(task, digest, rule, cached, dyn_score, maybe_part)
end
end
end
request_data.callback = vt_http_callback
http.request(request_data)
end
if common.condition_check_and_continue(task, content, rule, digest,
virustotal_check_uncached) then
return
else
virustotal_check_uncached()
end
end
return {
type = 'antivirus',
description = 'Virustotal integration',
configure = virustotal_config,
check = virustotal_check,
name = N
}
|
