1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
|
--[[[
-- @module lua_maps
-- This module contains helper functions for managing rspamd maps
--]]
--[[
Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--
local rspamd_logger = require "rspamd_logger"
local ts = require("tableshape").types
local lua_util = require "lua_util"
local exports = {}
local maps_cache = {}
local function map_hash_key(data, mtype)
local hash = require "rspamd_cryptobox_hash"
local st = hash.create_specific('xxh64')
st:update(data)
st:update(mtype)
return st:hex()
end
local function starts(where,st)
return string.sub(where,1,string.len(st))==st
end
local function cut_prefix(where,st)
return string.sub(where,#st + 1)
end
local function maybe_adjust_type(data,mtype)
local function check_prefix(prefix, t)
if starts(data, prefix) then
data = cut_prefix(data, prefix)
mtype = t
return true
end
return false
end
local known_types = {
{'regexp;', 'regexp'},
{'re;', 'regexp'},
{'regexp_multi;', 'regexp_multi'},
{'re_multi;', 'regexp_multi'},
{'glob;', 'glob'},
{'glob_multi;', 'glob_multi'},
{'radix;', 'radix'},
{'ipnet;', 'radix'},
{'set;', 'set'},
{'hash;', 'hash'},
{'plain;', 'hash'},
{'cdb;', 'cdb'},
{'cdb:/', 'cdb'},
}
if mtype == 'callback' then
return mtype
end
for _,t in ipairs(known_types) do
if check_prefix(t[1], t[2]) then
return data,mtype
end
end
-- No change
return data,mtype
end
--[[[
-- @function lua_maps.map_add_from_ucl(opt, mtype, description)
-- Creates a map from static data
-- Returns true if map was added or nil
-- @param {string or table} opt data for map (or URL)
-- @param {string} mtype type of map (`set`, `map`, `radix`, `regexp`)
-- @param {string} description human-readable description of map
-- @return {bool} true on success, or `nil`
--]]
local function rspamd_map_add_from_ucl(opt, mtype, description)
local ret = {
get_key = function(t, k)
if t.__data then
return t.__data:get_key(k)
end
return nil
end
}
local ret_mt = {
__index = function(t, k)
if t.__data then
return t.get_key(k)
end
return nil
end
}
if not opt then
return nil
end
if type(opt) == 'string' then
opt,mtype = maybe_adjust_type(opt, mtype)
local cache_key = map_hash_key(opt, mtype)
if maps_cache[cache_key] then
rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)',
opt, mtype)
return maps_cache[cache_key]
end
-- We have a single string, so we treat it as a map
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = opt,
}
if map then
ret.__data = map
ret.hash = cache_key
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
elseif type(opt) == 'table' then
local cache_key = lua_util.table_digest(opt)
if maps_cache[cache_key] then
rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s',
cache_key:sub(1,8), description)
return maps_cache[cache_key]
end
if opt[1] then
if mtype == 'radix' then
if string.find(opt[1], '^%d') then
local map = rspamd_config:radix_from_ucl(opt)
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
else
-- Plain table
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = opt,
}
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
end
elseif mtype == 'regexp' or mtype == 'glob' then
if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
-- Plain table
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = opt,
}
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
else
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = {
url = 'static',
data = opt,
}
}
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
end
else
if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
-- Plain table
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = opt,
}
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
else
local data = {}
local nelts = 0
-- Plain array of keys, count merely numeric elts
for _,elt in ipairs(opt) do
if type(elt) == 'string' then
-- Numeric table
if mtype == 'hash' then
-- Treat as KV pair
local pieces = lua_util.str_split(elt, ' ')
if #pieces > 1 then
local key = table.remove(pieces, 1)
data[key] = table.concat(pieces, ' ')
else
data[elt] = true
end
else
data[elt] = true
end
nelts = nelts + 1
end
end
if nelts > 0 then
-- Plain Lua table that is used as a map
ret.__data = data
ret.get_key = function(t, k)
if k ~= '__data' then
return t.__data[k]
end
return nil
end
maps_cache[cache_key] = ret
return ret
else
-- Empty map, huh?
rspamd_logger.errx(rspamd_config, 'invalid map element: %s',
opt)
end
end
end
else
-- We have some non-trivial object so let C code to deal with it somehow...
local map = rspamd_config:add_map{
type = mtype,
description = description,
url = opt,
}
if map then
ret.__data = map
setmetatable(ret, ret_mt)
maps_cache[cache_key] = ret
return ret
end
end -- opt[1]
end
return nil
end
--[[[
-- @function lua_maps.map_add(mname, optname, mtype, description)
-- Creates a map from configuration elements (static data or URL)
-- Returns true if map was added or nil
-- @param {string} mname config section to use
-- @param {string} optname option name to use
-- @param {string} mtype type of map ('set', 'hash', 'radix', 'regexp', 'glob')
-- @param {string} description human-readable description of map
-- @return {bool} true on success, or `nil`
--]]
local function rspamd_map_add(mname, optname, mtype, description)
local opt = rspamd_config:get_module_opt(mname, optname)
return rspamd_map_add_from_ucl(opt, mtype, description)
end
exports.rspamd_map_add = rspamd_map_add
exports.map_add = rspamd_map_add
exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl
exports.map_add_from_ucl = rspamd_map_add_from_ucl
-- Check `what` for being lua_map name, otherwise just compares key with what
local function rspamd_maybe_check_map(key, what)
local fun = require "fun"
if type(what) == "table" then
return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what)
end
if type(rspamd_maps) == "table" then
local mn
if starts(what, "map:") then
mn = string.sub(what, 4)
elseif starts(what, "map://") then
mn = string.sub(what, 6)
end
if mn and rspamd_maps[mn] then
return rspamd_maps[mn]:get_key(key)
else
return what:lower() == key
end
else
return what:lower() == key
end
end
exports.rspamd_maybe_check_map = rspamd_maybe_check_map
exports.map_schema = ts.one_of{
ts.string, -- 'http://some_map'
ts.array_of(ts.string), -- ['foo', 'bar']
ts.shape{ -- complex object
name = ts.string:is_optional(),
description = ts.string:is_optional(),
timeout = ts.number,
data = ts.array_of(ts.string):is_optional(),
-- Tableshape has no options support for something like key1 or key2?
upstreams = ts.one_of{
ts.string,
ts.array_of(ts.string),
}:is_optional(),
url = ts.one_of{
ts.string,
ts.array_of(ts.string),
}:is_optional(),
}
}
return exports
|