You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

known_senders.lua 7.8KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. --[[
  2. Copyright (c) 2023, Vsevolod Stakhov <vsevolod@rspamd.com>
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ]]--
  13. -- This plugin implements known senders logic for Rspamd
  14. local rspamd_logger = require "rspamd_logger"
  15. local ts = (require "tableshape").types
  16. local N = 'known_senders'
  17. local lua_util = require "lua_util"
  18. local lua_redis = require "lua_redis"
  19. local lua_maps = require "lua_maps"
  20. local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
  21. if confighelp then
  22. rspamd_config:add_example(nil, 'known_senders',
  23. "Maintain a list of known senders using Redis",
  24. [[
  25. known_senders {
  26. # Domains to track senders
  27. domains = "https://maps.rspamd.com/freemail/free.txt.zst";
  28. # Maximum number of elements
  29. max_senders = 100000;
  30. # Maximum time to live (when not using bloom filters)
  31. max_ttl = 30d;
  32. # Use bloom filters (must be enabled in Redis as a plugin)
  33. use_bloom = false;
  34. # Insert symbol for new senders from the specific domains
  35. symbol_unknown = 'UNKNOWN_SENDER';
  36. }
  37. ]])
  38. return
  39. end
  40. local redis_params
  41. local settings = {
  42. domains = {},
  43. max_senders = 100000,
  44. max_ttl = 30 * 86400,
  45. use_bloom = false,
  46. symbol = 'KNOWN_SENDER',
  47. symbol_unknown = 'UNKNOWN_SENDER',
  48. redis_key = 'rs_known_senders',
  49. }
  50. local settings_schema = lua_redis.enrich_schema({
  51. domains = lua_maps.map_schema,
  52. enabled = ts.boolean:is_optional(),
  53. max_senders = (ts.integer + ts.string / tonumber):is_optional(),
  54. max_ttl = (ts.integer + ts.string / tonumber):is_optional(),
  55. use_bloom = ts.boolean:is_optional(),
  56. redis_key = ts.string:is_optional(),
  57. symbol = ts.string:is_optional(),
  58. symbol_unknown = ts.string:is_optional(),
  59. })
  60. local function make_key(input)
  61. local hash = rspamd_cryptobox_hash.create_specific('md5')
  62. hash:update(input.addr)
  63. return hash:hex()
  64. end
  65. local function check_redis_key(task, key, key_ty)
  66. lua_util.debugm(N, task, 'check key %s, type: %s', key, key_ty)
  67. local function redis_zset_callback(err, data)
  68. lua_util.debugm(N, task, 'got data: %s', data)
  69. if err then
  70. rspamd_logger.errx(task, 'redis error: %s', err)
  71. elseif data then
  72. if type(data) ~= 'userdata' then
  73. -- non-null reply
  74. task:insert_result(settings.symbol, 1.0, string.format("%s:%s", key_ty, key))
  75. else
  76. if settings.symbol_unknown then
  77. task:insert_result(settings.symbol_unknown, 1.0, string.format("%s:%s", key_ty, key))
  78. end
  79. lua_util.debugm(N, task, 'insert key %s, type: %s', key, key_ty)
  80. -- Insert key to zset and trim it's cardinality
  81. lua_redis.redis_make_request(task,
  82. redis_params, -- connect params
  83. key, -- hash key
  84. true, -- is write
  85. nil, --callback
  86. 'ZADD', -- command
  87. { settings.redis_key, tostring(task:get_timeval(true)), key } -- arguments
  88. )
  89. lua_redis.redis_make_request(task,
  90. redis_params, -- connect params
  91. key, -- hash key
  92. true, -- is write
  93. nil, --callback
  94. 'ZREMRANGEBYRANK', -- command
  95. { settings.redis_key, '0',
  96. tostring(-(settings.max_senders + 1)) } -- arguments
  97. )
  98. end
  99. end
  100. end
  101. local function redis_bloom_callback(err, data)
  102. lua_util.debugm(N, task, 'got data: %s', data)
  103. if err then
  104. rspamd_logger.errx(task, 'redis error: %s', err)
  105. elseif data then
  106. if type(data) ~= 'userdata' and data == 1 then
  107. -- non-null reply equal to `1`
  108. task:insert_result(settings.symbol, 1.0, string.format("%s:%s", key_ty, key))
  109. else
  110. if settings.symbol_unknown then
  111. task:insert_result(settings.symbol_unknown, 1.0, string.format("%s:%s", key_ty, key))
  112. end
  113. lua_util.debugm(N, task, 'insert key %s, type: %s', key, key_ty)
  114. -- Reserve bloom filter space
  115. lua_redis.redis_make_request(task,
  116. redis_params, -- connect params
  117. key, -- hash key
  118. true, -- is write
  119. nil, --callback
  120. 'BF.RESERVE', -- command
  121. { settings.redis_key, tostring(settings.max_senders), '0.01', '1000', 'NONSCALING' } -- arguments
  122. )
  123. -- Insert key and adjust bloom filter
  124. lua_redis.redis_make_request(task,
  125. redis_params, -- connect params
  126. key, -- hash key
  127. true, -- is write
  128. nil, --callback
  129. 'BF.ADD', -- command
  130. { settings.redis_key, key } -- arguments
  131. )
  132. end
  133. end
  134. end
  135. if settings.use_bloom then
  136. lua_redis.redis_make_request(task,
  137. redis_params, -- connect params
  138. key, -- hash key
  139. false, -- is write
  140. redis_bloom_callback, --callback
  141. 'BF.EXISTS', -- command
  142. { settings.redis_key, key } -- arguments
  143. )
  144. else
  145. lua_redis.redis_make_request(task,
  146. redis_params, -- connect params
  147. key, -- hash key
  148. false, -- is write
  149. redis_zset_callback, --callback
  150. 'ZSCORE', -- command
  151. { settings.redis_key, key } -- arguments
  152. )
  153. end
  154. end
  155. local function known_senders_callback(task)
  156. local mime_from = (task:get_from('mime') or {})[1]
  157. local smtp_from = (task:get_from('smtp') or {})[1]
  158. local mime_key, smtp_key
  159. if mime_from and mime_from.addr then
  160. if settings.domains:get_key(mime_from.domain) then
  161. mime_key = make_key(mime_from)
  162. else
  163. lua_util.debugm(N, task, 'skip mime from domain %s', mime_from.domain)
  164. end
  165. end
  166. if smtp_from and smtp_from.addr then
  167. if settings.domains:get_key(smtp_from.domain) then
  168. smtp_key = make_key(smtp_from)
  169. else
  170. lua_util.debugm(N, task, 'skip smtp from domain %s', smtp_from.domain)
  171. end
  172. end
  173. if mime_key and smtp_key and mime_key ~= smtp_key then
  174. -- Check both keys
  175. check_redis_key(task, mime_key, 'mime')
  176. check_redis_key(task, smtp_key, 'smtp')
  177. elseif mime_key then
  178. -- Check mime key
  179. check_redis_key(task, mime_key, 'mime')
  180. elseif smtp_key then
  181. -- Check smtp key
  182. check_redis_key(task, smtp_key, 'smtp')
  183. end
  184. end
  185. local opts = rspamd_config:get_all_opt('known_senders')
  186. if opts then
  187. settings = lua_util.override_defaults(settings, opts)
  188. local res, err = settings_schema:transform(settings)
  189. if not res then
  190. rspamd_logger.errx(rspamd_config, 'cannot parse known_senders options: %1', err)
  191. else
  192. settings = res
  193. end
  194. redis_params = lua_redis.parse_redis_server(N, opts)
  195. if redis_params then
  196. local map_conf = settings.domains
  197. settings.domains = lua_maps.map_add_from_ucl(settings.domains, 'set', 'domains to track senders from')
  198. if not settings.domains then
  199. rspamd_logger.errx(rspamd_config, "couldn't add map %s, disable module",
  200. map_conf)
  201. lua_util.disable_module(N, "config")
  202. return
  203. end
  204. lua_redis.register_prefix(settings.redis_key, N,
  205. 'Known elements redis key', {
  206. type = 'zset/bloom filter',
  207. })
  208. local id = rspamd_config:register_symbol({
  209. name = settings.symbol,
  210. type = 'normal',
  211. callback = known_senders_callback,
  212. one_shot = true,
  213. score = -1.0,
  214. augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
  215. })
  216. if settings.symbol_unknown and #settings.symbol_unknown > 0 then
  217. rspamd_config:register_symbol({
  218. name = settings.symbol_unknown,
  219. type = 'virtual',
  220. parent = id,
  221. one_shot = true,
  222. score = 0.5,
  223. })
  224. end
  225. else
  226. lua_util.disable_module(N, "redis")
  227. end
  228. end