1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
|
--[[
Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--
--[[[
-- @module lua_magic
-- This module contains file types detection logic
--]]
local patterns = require "lua_magic/patterns"
local types = require "lua_magic/types"
local heuristics = require "lua_magic/heuristics"
local fun = require "fun"
local lua_util = require "lua_util"
local rspamd_text = require "rspamd_text"
local rspamd_trie = require "rspamd_trie"
local N = "lua_magic"
local exports = {}
-- trie objects
local compiled_patterns
local compiled_short_patterns
local compiled_tail_patterns
-- {<str>, <match_object>, <pattern_object>} indexed by pattern number
local processed_patterns = {}
local short_patterns = {}
local tail_patterns = {}
local short_match_limit = 128
local max_short_offset = -1
local min_tail_offset = math.huge
local function process_patterns(log_obj)
-- Add pattern to either short patterns or to normal patterns
local function add_processed(str, match, pattern)
if match.position and type(match.position) == 'number' then
if match.tail then
-- Tail pattern
tail_patterns[#tail_patterns + 1] = {
str, match, pattern
}
if min_tail_offset > match.tail then
min_tail_offset = match.tail
end
lua_util.debugm(N, log_obj, 'add tail pattern %s for ext %s',
str, pattern.ext)
elseif match.position < short_match_limit then
short_patterns[#short_patterns + 1] = {
str, match, pattern
}
lua_util.debugm(N, log_obj, 'add short pattern %s for ext %s',
str, pattern.ext)
if max_short_offset < match.position then
max_short_offset = match.position
end
else
processed_patterns[#processed_patterns + 1] = {
str, match, pattern
}
lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
str, pattern.ext)
end
else
processed_patterns[#processed_patterns + 1] = {
str, match, pattern
}
lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
str, pattern.ext)
end
end
if not compiled_patterns then
for ext,pattern in pairs(patterns) do
assert(types[ext], 'not found type: ' .. ext)
pattern.ext = ext
for _,match in ipairs(pattern.matches) do
if match.string then
if match.relative_position and not match.position then
match.position = match.relative_position + #match.string
end
add_processed(match.string, match, pattern)
elseif match.hex then
local hex_table = {}
for i=1,#match.hex,2 do
local subc = match.hex:sub(i, i + 1)
hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
end
if match.relative_position and not match.position then
match.position = match.relative_position + #match.hex / 2
end
add_processed(table.concat(hex_table), match, pattern)
end
end
end
compiled_patterns = rspamd_trie.create(fun.totable(
fun.map(function(t) return t[1] end, processed_patterns)),
rspamd_trie.flags.re
)
compiled_short_patterns = rspamd_trie.create(fun.totable(
fun.map(function(t) return t[1] end, short_patterns)),
rspamd_trie.flags.re
)
compiled_tail_patterns = rspamd_trie.create(fun.totable(
fun.map(function(t) return t[1] end, tail_patterns)),
rspamd_trie.flags.re
)
lua_util.debugm(N, log_obj,
'compiled %s (%s short; %s long; %s tail) patterns',
#processed_patterns + #short_patterns + #tail_patterns,
#short_patterns, #processed_patterns, #tail_patterns)
end
end
process_patterns(rspamd_config)
local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_obj, res)
local matches = trie:match(chunk)
local last = tlen
local function add_result(weight, ext)
if not res[ext] then
res[ext] = 0
end
if weight then
res[ext] = res[ext] + weight
else
res[ext] = res[ext] + 1
end
lua_util.debugm(N, log_obj,'add pattern for %s, weight %s, total weight %s',
ext, weight, res[ext])
end
local function match_position(pos, expected)
local cmp = function(a, b) return a == b end
if type(expected) == 'table' then
-- Something like {'>', 0}
if expected[1] == '>' then
cmp = function(a, b) return a > b end
elseif expected[1] == '>=' then
cmp = function(a, b) return a >= b end
elseif expected[1] == '<' then
cmp = function(a, b) return a < b end
elseif expected[1] == '<=' then
cmp = function(a, b) return a <= b end
elseif expected[1] == '!=' then
cmp = function(a, b) return a ~= b end
end
expected = expected[2]
end
-- Tail match
if expected < 0 then
expected = last + expected + 1
end
return cmp(pos, expected)
end
for npat,matched_positions in pairs(matches) do
local pat_data = processed_tbl[npat]
local pattern = pat_data[3]
local match = pat_data[2]
-- Single position
if match.position then
local position = match.position
for _,pos in ipairs(matched_positions) do
lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
pattern.ext, pos, offset)
if match_position(pos + offset, position) then
if match.heuristic then
local ext,weight = match.heuristic(input, log_obj)
if ext then
add_result(weight, ext)
break
end
else
add_result(match.weight, pattern.ext)
break
end
end
end
elseif match.positions then
-- Match all positions
local all_right = true
for _,position in ipairs(match.positions) do
local matched = false
for _,pos in ipairs(matched_positions) do
if not match_position(pos + offset, position) then
matched = true
break
end
end
if not matched then
all_right = false
break
end
end
if all_right then
if match.heuristic then
local ext,weight = match.heuristic(input, log_obj)
if ext then
add_result(weight, ext)
break
end
else
add_result(match.weight, pattern.ext)
break
end
end
end
end
end
local function process_detected(res)
local extensions = lua_util.keys(res)
if #extensions > 0 then
table.sort(extensions, function(ex1, ex2)
return res[ex1] > res[ex2]
end)
return extensions,res[extensions[1]]
end
return nil
end
exports.detect = function(input, log_obj)
if not log_obj then log_obj = rspamd_config end
local res = {}
if type(input) == 'string' then
-- Convert to rspamd_text
input = rspamd_text.fromstring(input)
end
if type(input) == 'userdata' then
local inplen = #input
-- Check tail matches
if inplen > min_tail_offset then
local tail = input:span(inplen - min_tail_offset, min_tail_offset)
match_chunk(tail, input, inplen, inplen - min_tail_offset,
compiled_tail_patterns, tail_patterns, log_obj, res)
end
-- Try short match
local head = input:span(1, math.min(max_short_offset, inplen))
match_chunk(head, input, inplen, 0,
compiled_short_patterns, short_patterns, log_obj, res)
-- Check if we have enough data or go to long patterns
local extensions,confidence = process_detected(res)
if extensions and #extensions > 0 and confidence > 30 then
-- We are done on short patterns
return extensions[1],types[extensions[1]]
end
-- No way, let's check data in chunks or just the whole input if it is small enough
if #input > exports.chunk_size * 3 then
-- Chunked version as input is too long
local chunk1, chunk2 =
input:span(1, exports.chunk_size * 2),
input:span(inplen - exports.chunk_size, exports.chunk_size)
local offset1, offset2 = 0, inplen - exports.chunk_size
match_chunk(chunk1, input, inplen,
offset1, compiled_patterns, processed_patterns, log_obj, res)
match_chunk(chunk2, input, inplen,
offset2, compiled_patterns, processed_patterns, log_obj, res)
else
-- Input is short enough to match it at all
match_chunk(input, input, inplen, 0,
compiled_patterns, processed_patterns, log_obj, res)
end
else
-- Table input is NYI
assert(0, 'table input for match')
end
local extensions = process_detected(res)
if extensions and #extensions > 0 then
return extensions[1],types[extensions[1]]
end
-- Nothing found
return nil
end
exports.detect_mime_part = function(part, log_obj)
local ext,weight = heuristics.mime_part_heuristic(part, log_obj)
if ext and weight and weight > 20 then
return ext,types[ext]
end
ext,weight = exports.detect(part:get_content(), log_obj)
if ext and weight and weight > 20 then
return ext,types[ext]
end
-- Text/html and other parts
ext,weight = heuristics.text_part_heuristic(part, log_obj)
if ext and weight and weight > 20 then
return ext,types[ext]
end
end
-- This parameter specifies how many bytes are checked in the input
-- Rspamd checks 2 chunks at start and 1 chunk at the end
exports.chunk_size = 32768
return exports
|