--[[ | |||||
Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> | |||||
Licensed under the Apache License, Version 2.0 (the "License"); | |||||
you may not use this file except in compliance with the License. | |||||
You may obtain a copy of the License at | |||||
http://www.apache.org/licenses/LICENSE-2.0 | |||||
Unless required by applicable law or agreed to in writing, software | |||||
distributed under the License is distributed on an "AS IS" BASIS, | |||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
See the License for the specific language governing permissions and | |||||
limitations under the License. | |||||
]]-- | |||||
local exports = {} | |||||
local logger = require 'rspamd_logger' | |||||
local lua_util = require 'lua_util' | |||||
local squeezed_rules = {{}} -- plain vector of all rules squeezed | |||||
local squeezed_symbols = {} -- indexed by name of symbol | |||||
local squeezed_deps = {} -- squeezed deps | |||||
local squeezed_rdeps = {} -- squeezed reverse deps | |||||
local SN = 'lua_squeeze' | |||||
local squeeze_sym = 'LUA_SQUEEZE' | |||||
local squeeze_function_ids = {} | |||||
local squeezed_groups = {} | |||||
local last_rule | |||||
local virtual_symbols = {} | |||||
local function gen_lua_squeeze_function(order) | |||||
return function(task) | |||||
local symbols_disabled = task:cache_get('squeezed_disable') | |||||
local mime_task = task:has_flag('mime') | |||||
for _,data in ipairs(squeezed_rules[order]) do | |||||
local disable = false | |||||
if symbols_disabled and symbols_disabled[data[2]] then | |||||
disable = true | |||||
end | |||||
if data[3] and data[3].flags.mime then | |||||
if not mime_task then | |||||
disable = true | |||||
end | |||||
end | |||||
if not disable then | |||||
local function real_call() | |||||
return {data[1](task)} | |||||
end | |||||
-- Too expensive to call :( | |||||
lua_util.debugm(SN, task, 'call for: %s', data[2]) | |||||
local status, ret = pcall(real_call) | |||||
if not status then | |||||
logger.errx(task, 'error in squeezed rule %s: %s', data[2], ret) | |||||
else | |||||
if #ret ~= 0 then | |||||
local first = ret[1] | |||||
local sym = data[2] | |||||
-- Function has returned something, so it is rule, not a plugin | |||||
if type(first) == 'boolean' then | |||||
if first then | |||||
table.remove(ret, 1) | |||||
local second = ret[1] | |||||
if type(second) == 'number' then | |||||
table.remove(ret, 1) | |||||
if second ~= 0 then | |||||
if type(ret[1]) == 'table' then | |||||
task:insert_result(sym, second, ret[1]) | |||||
else | |||||
task:insert_result(sym, second, ret) | |||||
end | |||||
end | |||||
else | |||||
if type(ret[1]) == 'table' then | |||||
task:insert_result(sym, 1.0, ret[1]) | |||||
else | |||||
task:insert_result(sym, 1.0, ret) | |||||
end | |||||
end | |||||
end | |||||
elseif type(first) == 'number' then | |||||
table.remove(ret, 1) | |||||
if first ~= 0 then | |||||
if type(ret[1]) == 'table' then | |||||
task:insert_result(sym, first, ret[1]) | |||||
else | |||||
task:insert_result(sym, first, ret) | |||||
end | |||||
end | |||||
else | |||||
if type(ret[1]) == 'table' then | |||||
task:insert_result(sym, 1.0, ret[1]) | |||||
else | |||||
task:insert_result(sym, 1.0, ret) | |||||
end | |||||
end | |||||
end | |||||
end | |||||
else | |||||
lua_util.debugm(SN, task, 'skip symbol due to settings: %s', data[2]) | |||||
end | |||||
end | |||||
end | |||||
end | |||||
exports.squeeze_rule = function(s, func, flags) | |||||
if s then | |||||
if not squeezed_symbols[s] then | |||||
squeezed_symbols[s] = { | |||||
cb = func, | |||||
order = 0, | |||||
sym = s, | |||||
flags = flags or {} | |||||
} | |||||
lua_util.debugm(SN, rspamd_config, 'squeezed rule: %s', s) | |||||
else | |||||
logger.warnx(rspamd_config, 'duplicate symbol registered: %s, skip', s) | |||||
end | |||||
else | |||||
-- Unconditionally add function to the squeezed rules | |||||
local id = tostring(#squeezed_rules) | |||||
lua_util.debugm(SN, rspamd_config, 'squeezed unnamed rule: %s', id) | |||||
table.insert(squeezed_rules[1], {func, 'unnamed: ' .. id, squeezed_symbols[s]}) | |||||
end | |||||
if not squeeze_function_ids[1] then | |||||
squeeze_function_ids[1] = rspamd_config:register_symbol{ | |||||
type = 'callback', | |||||
flags = 'squeezed', | |||||
callback = gen_lua_squeeze_function(1), | |||||
name = squeeze_sym, | |||||
description = 'Meta rule for Lua rules that can be squeezed', | |||||
no_squeeze = true, -- to avoid infinite recursion | |||||
} | |||||
end | |||||
last_rule = s | |||||
return squeeze_function_ids[1] | |||||
end | |||||
exports.squeeze_virtual = function(id, name) | |||||
if squeeze_function_ids[1] and id == squeeze_function_ids[1] and last_rule then | |||||
virtual_symbols[name] = last_rule | |||||
lua_util.debugm(SN, rspamd_config, 'add virtual symbol %s -> %s', | |||||
name, last_rule) | |||||
return id | |||||
end | |||||
return -1 | |||||
end | |||||
exports.squeeze_dependency = function(child, parent) | |||||
lua_util.debugm(SN, rspamd_config, 'squeeze dep %s->%s', child, parent) | |||||
if not squeezed_deps[parent] then | |||||
squeezed_deps[parent] = {} | |||||
end | |||||
if not squeezed_deps[parent][child] then | |||||
squeezed_deps[parent][child] = true | |||||
else | |||||
logger.warnx(rspamd_config, 'duplicate dependency %s->%s', child, parent) | |||||
end | |||||
if not squeezed_rdeps[child] then | |||||
squeezed_rdeps[child] = {} | |||||
end | |||||
if not squeezed_rdeps[child][parent] then | |||||
squeezed_rdeps[child][parent] = true | |||||
end | |||||
return true | |||||
end | |||||
local function get_ordered_symbol_name(order) | |||||
if order == 1 then | |||||
return squeeze_sym | |||||
end | |||||
return squeeze_sym .. tostring(order) | |||||
end | |||||
local function register_topology_symbol(order) | |||||
local ord_sym = get_ordered_symbol_name(order) | |||||
squeeze_function_ids[order] = rspamd_config:register_symbol{ | |||||
type = 'callback', | |||||
flags = 'squeezed', | |||||
callback = gen_lua_squeeze_function(order), | |||||
name = ord_sym, | |||||
description = 'Meta rule for Lua rules that can be squeezed, order ' .. tostring(order), | |||||
no_squeeze = true, -- to avoid infinite recursion | |||||
} | |||||
local parent = get_ordered_symbol_name(order - 1) | |||||
lua_util.debugm(SN, rspamd_config, 'registered new order of deps: %s->%s', | |||||
ord_sym, parent) | |||||
rspamd_config:register_dependency(ord_sym, parent, true) | |||||
end | |||||
exports.squeeze_init = function() | |||||
-- Do topological sorting | |||||
for _,v in pairs(squeezed_symbols) do | |||||
local function visit(node, order) | |||||
if order > node.order then | |||||
node.order = order | |||||
lua_util.debugm(SN, rspamd_config, "symbol: %s, order: %s", node.sym, order) | |||||
else | |||||
return | |||||
end | |||||
if squeezed_deps[node.sym] then | |||||
for dep,_ in pairs(squeezed_deps[node.sym]) do | |||||
if squeezed_symbols[dep] then | |||||
visit(squeezed_symbols[dep], order + 1) | |||||
end | |||||
end | |||||
end | |||||
end | |||||
if v.order == 0 then | |||||
visit(v, 1) | |||||
end | |||||
end | |||||
for parent,children in pairs(squeezed_deps) do | |||||
if not squeezed_symbols[parent] then | |||||
local real_parent = virtual_symbols[parent] | |||||
if real_parent then | |||||
parent = real_parent | |||||
end | |||||
end | |||||
if not squeezed_symbols[parent] then | |||||
-- Trivial case, external dependnency | |||||
for s,_ in pairs(children) do | |||||
if squeezed_symbols[s] then | |||||
-- External dep depends on a squeezed symbol | |||||
lua_util.debugm(SN, rspamd_config, 'register external squeezed dependency on %s', | |||||
parent) | |||||
rspamd_config:register_dependency(squeeze_sym, parent, true) | |||||
else | |||||
-- Generic rspamd symbols dependency | |||||
lua_util.debugm(SN, rspamd_config, 'register external dependency %s -> %s', | |||||
s, parent) | |||||
rspamd_config:register_dependency(s, parent, true) | |||||
end | |||||
end | |||||
else | |||||
-- Not so trivial case | |||||
local ps = squeezed_symbols[parent] | |||||
for cld,_ in pairs(children) do | |||||
if squeezed_symbols[cld] then | |||||
-- Cross dependency | |||||
lua_util.debugm(SN, rspamd_config, 'cross dependency in squeezed symbols %s->%s', | |||||
cld, parent) | |||||
local order = squeezed_symbols[cld].order | |||||
if not squeeze_function_ids[order] then | |||||
-- Need to register new callback symbol to handle deps | |||||
for i = 1, order do | |||||
if not squeeze_function_ids[i] then | |||||
register_topology_symbol(i) | |||||
end | |||||
end | |||||
end | |||||
else | |||||
-- External symbol depends on a squeezed one | |||||
local parent_symbol = get_ordered_symbol_name(ps.order) | |||||
rspamd_config:register_dependency(cld, parent_symbol, true) | |||||
lua_util.debugm(SN, rspamd_config, 'register squeezed dependency for external symbol %s->%s', | |||||
cld, parent_symbol) | |||||
end | |||||
end | |||||
end | |||||
end | |||||
-- We have now all deps being registered, so we can register virtual symbols | |||||
-- and create squeezed rules | |||||
for k,v in pairs(squeezed_symbols) do | |||||
local parent_symbol = get_ordered_symbol_name(v.order) | |||||
lua_util.debugm(SN, rspamd_config, 'added squeezed rule: %s (%s): %s', | |||||
k, parent_symbol, v) | |||||
rspamd_config:register_symbol{ | |||||
type = 'virtual', | |||||
name = k, | |||||
flags = 'squeezed', | |||||
parent = squeeze_function_ids[v.order], | |||||
no_squeeze = true, -- to avoid infinite recursion | |||||
} | |||||
local metric_sym = rspamd_config:get_metric_symbol(k) | |||||
if metric_sym then | |||||
v.group = metric_sym.group | |||||
v.groups = metric_sym.groups | |||||
v.score = metric_sym.score | |||||
v.description = metric_sym.description | |||||
if v.group then | |||||
if not squeezed_groups[v.group] then | |||||
lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) | |||||
squeezed_groups[v.group] = {} | |||||
end | |||||
table.insert(squeezed_groups[v.group], k) | |||||
end | |||||
if v.groups then | |||||
for _,gr in ipairs(v.groups) do | |||||
if not squeezed_groups[gr] then | |||||
lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', gr) | |||||
squeezed_groups[gr] = {} | |||||
end | |||||
table.insert(squeezed_groups[gr], k) | |||||
end | |||||
end | |||||
else | |||||
lua_util.debugm(SN, rspamd_config, 'no metric symbol found for %s, maybe bug', k) | |||||
end | |||||
if not squeezed_rules[v.order] then | |||||
squeezed_rules[v.order] = {} | |||||
end | |||||
table.insert(squeezed_rules[v.order], {v.cb,k,v}) | |||||
end | |||||
end | |||||
exports.handle_settings = function(task, settings) | |||||
local symbols_disabled = {} | |||||
local symbols_enabled = {} | |||||
local found = false | |||||
local disabled = false | |||||
if settings.default then settings = settings.default end | |||||
local function disable_all() | |||||
for k,sym in pairs(squeezed_symbols) do | |||||
if not symbols_enabled[k] and not (sym.flags and sym.flags.explicit_disable) then | |||||
symbols_disabled[k] = true | |||||
end | |||||
end | |||||
end | |||||
if settings.symbols_enabled then | |||||
disable_all() | |||||
found = true | |||||
disabled = true | |||||
for _,s in ipairs(settings.symbols_enabled) do | |||||
if squeezed_symbols[s] then | |||||
lua_util.debugm(SN, task, 'enable symbol %s as it is in `symbols_enabled`', s) | |||||
symbols_enabled[s] = true | |||||
symbols_disabled[s] = nil | |||||
end | |||||
end | |||||
end | |||||
if settings.groups_enabled then | |||||
if not disabled then | |||||
disable_all() | |||||
end | |||||
found = true | |||||
for _,gr in ipairs(settings.groups_enabled) do | |||||
if squeezed_groups[gr] then | |||||
for _,sym in ipairs(squeezed_groups[gr]) do | |||||
lua_util.debugm(SN, task, 'enable symbol %s as it is in `groups_enabled`', sym) | |||||
symbols_enabled[sym] = true | |||||
symbols_disabled[sym] = nil | |||||
end | |||||
end | |||||
end | |||||
end | |||||
if settings.symbols_disabled then | |||||
found = true | |||||
for _,s in ipairs(settings.symbols_disabled) do | |||||
lua_util.debugm(SN, task, 'try disable symbol %s as it is in `symbols_disabled`', s) | |||||
if not symbols_enabled[s] then | |||||
symbols_disabled[s] = true | |||||
lua_util.debugm(SN, task, 'disable symbol %s as it is in `symbols_disabled`', s) | |||||
end | |||||
end | |||||
end | |||||
if settings.groups_disabled then | |||||
found = true | |||||
for _,gr in ipairs(settings.groups_disabled) do | |||||
lua_util.debugm(SN, task, 'try disable group %s as it is in `groups_disabled`: %s', gr) | |||||
if squeezed_groups[gr] then | |||||
for _,sym in ipairs(squeezed_groups[gr]) do | |||||
if not symbols_enabled[sym] then | |||||
lua_util.debugm(SN, task, 'disable symbol %s as it is in `groups_disabled`', sym) | |||||
symbols_disabled[sym] = true | |||||
end | |||||
end | |||||
end | |||||
end | |||||
end | |||||
if found then | |||||
task:cache_set('squeezed_disable', symbols_disabled) | |||||
end | |||||
end | |||||
return exports |
gboolean enable_sessions_cache; /**< Enable session cache for debug */ | gboolean enable_sessions_cache; /**< Enable session cache for debug */ | ||||
gboolean enable_experimental; /**< Enable experimental plugins */ | gboolean enable_experimental; /**< Enable experimental plugins */ | ||||
gboolean disable_pcre_jit; /**< Disable pcre JIT */ | gboolean disable_pcre_jit; /**< Disable pcre JIT */ | ||||
gboolean disable_lua_squeeze; /**< Disable lua rules squeezing */ | |||||
gboolean own_lua_state; /**< True if we have created lua_state internally */ | gboolean own_lua_state; /**< True if we have created lua_state internally */ | ||||
gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ | gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ | ||||
G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit), | G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit), | ||||
0, | 0, | ||||
"Disable PCRE JIT"); | "Disable PCRE JIT"); | ||||
rspamd_rcl_add_default_handler (sub, | |||||
"disable_lua_squeeze", | |||||
rspamd_rcl_parse_struct_boolean, | |||||
G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze), | |||||
0, | |||||
"Disable Lua rules squeezing"); | |||||
rspamd_rcl_add_default_handler (sub, | rspamd_rcl_add_default_handler (sub, | ||||
"min_word_len", | "min_word_len", | ||||
rspamd_rcl_parse_struct_integer, | rspamd_rcl_parse_struct_integer, |
} | } | ||||
if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) { | if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) { | ||||
lua_State *L = cfg->lua_state; | |||||
int err_idx; | |||||
/* Process squeezed Lua rules */ | |||||
lua_pushcfunction (L, &rspamd_lua_traceback); | |||||
err_idx = lua_gettop (L); | |||||
if (rspamd_lua_require_function (cfg->lua_state, "lua_squeeze_rules", | |||||
"squeeze_init")) { | |||||
if (lua_pcall (L, 0, 0, err_idx) != 0) { | |||||
msg_err_config ("call to squeeze_init script failed: %s", | |||||
lua_tostring (L, -1)); | |||||
} | |||||
} | |||||
lua_settop (L, err_idx - 1); | |||||
/* Init config cache */ | /* Init config cache */ | ||||
rspamd_symcache_init (cfg->cache); | rspamd_symcache_init (cfg->cache); | ||||
(dyn_item)->finished = 1 | (dyn_item)->finished = 1 | ||||
#define CLR_FINISH_BIT(checkpoint, dyn_item) \ | #define CLR_FINISH_BIT(checkpoint, dyn_item) \ | ||||
(dyn_item)->finished = 0 | (dyn_item)->finished = 0 | ||||
static const guchar rspamd_symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 }; | static const guchar rspamd_symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 }; | ||||
struct rspamd_symcache_header { | struct rspamd_symcache_header { | ||||
ref_entry_t ref; | ref_entry_t ref; | ||||
}; | }; | ||||
/* | |||||
* This structure is optimised to store ids list: | |||||
* - If the first element is -1 then use dynamic part, else use static part | |||||
*/ | |||||
struct rspamd_symcache_id_list { | |||||
union { | |||||
guint32 st[4]; | |||||
struct { | |||||
guint32 e; | |||||
guint32 dynlen; | |||||
guint *n; | |||||
} dyn; | |||||
}; | |||||
}; | |||||
struct rspamd_symcache_item { | struct rspamd_symcache_item { | ||||
/* This block is likely shared */ | /* This block is likely shared */ | ||||
struct item_stat *st; | struct item_stat *st; | ||||
guint order; | guint order; | ||||
gint id; | gint id; | ||||
gint frequency_peaks; | gint frequency_peaks; | ||||
/* Settings ids */ | |||||
struct rspamd_symcache_id_list allowed_ids; | |||||
struct rspamd_symcache_id_list forbidden_ids; | |||||
/* Dependencies */ | /* Dependencies */ | ||||
GPtrArray *deps; | GPtrArray *deps; | ||||
GPtrArray *composites; | GPtrArray *composites; | ||||
GPtrArray *idempotent; | GPtrArray *idempotent; | ||||
GPtrArray *virtual; | GPtrArray *virtual; | ||||
GPtrArray *squeezed; | |||||
GList *delayed_deps; | GList *delayed_deps; | ||||
GList *delayed_conditions; | GList *delayed_conditions; | ||||
rspamd_mempool_t *static_pool; | rspamd_mempool_t *static_pool; | ||||
} | } | ||||
} | } | ||||
if (item->type & SYMBOL_TYPE_SQUEEZED) { | |||||
g_ptr_array_add (cache->squeezed, item); | |||||
} | |||||
cache->used_items ++; | cache->used_items ++; | ||||
cache->id ++; | cache->id ++; | ||||
g_ptr_array_free (cache->postfilters, TRUE); | g_ptr_array_free (cache->postfilters, TRUE); | ||||
g_ptr_array_free (cache->idempotent, TRUE); | g_ptr_array_free (cache->idempotent, TRUE); | ||||
g_ptr_array_free (cache->composites, TRUE); | g_ptr_array_free (cache->composites, TRUE); | ||||
g_ptr_array_free (cache->squeezed, TRUE); | |||||
REF_RELEASE (cache->items_by_order); | REF_RELEASE (cache->items_by_order); | ||||
if (cache->peak_cb != -1) { | if (cache->peak_cb != -1) { | ||||
cache->idempotent = g_ptr_array_new (); | cache->idempotent = g_ptr_array_new (); | ||||
cache->composites = g_ptr_array_new (); | cache->composites = g_ptr_array_new (); | ||||
cache->virtual = g_ptr_array_new (); | cache->virtual = g_ptr_array_new (); | ||||
cache->squeezed = g_ptr_array_new (); | |||||
cache->reload_time = cfg->cache_reload_time; | cache->reload_time = cfg->cache_reload_time; | ||||
cache->total_hits = 1; | cache->total_hits = 1; | ||||
cache->total_weight = 1.0; | cache->total_weight = 1.0; | ||||
PTR_ARRAY_FOREACH (cache->items_by_id, i, item) { | PTR_ARRAY_FOREACH (cache->items_by_id, i, item) { | ||||
dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | ||||
if (!(item->type & (SYMBOL_TYPE_SQUEEZED|skip_mask))) { | |||||
if (!(item->type & (skip_mask))) { | |||||
SET_FINISH_BIT (checkpoint, dyn_item); | SET_FINISH_BIT (checkpoint, dyn_item); | ||||
SET_START_BIT (checkpoint, dyn_item); | SET_START_BIT (checkpoint, dyn_item); | ||||
} | } | ||||
item = rspamd_symcache_find_filter (cache, symbol); | item = rspamd_symcache_find_filter (cache, symbol); | ||||
if (item) { | if (item) { | ||||
if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { | |||||
dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | |||||
SET_FINISH_BIT (checkpoint, dyn_item); | |||||
SET_START_BIT (checkpoint, dyn_item); | |||||
msg_debug_cache_task ("disable execution of %s", symbol); | |||||
} | |||||
else { | |||||
msg_debug_cache_task ("skip disabling squeezed symbol %s", symbol); | |||||
} | |||||
dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | |||||
SET_FINISH_BIT (checkpoint, dyn_item); | |||||
SET_START_BIT (checkpoint, dyn_item); | |||||
msg_debug_cache_task ("disable execution of %s", symbol); | |||||
} | } | ||||
else { | else { | ||||
msg_info_task ("cannot disable %s: not found", symbol); | msg_info_task ("cannot disable %s: not found", symbol); | ||||
item = rspamd_symcache_find_filter (cache, symbol); | item = rspamd_symcache_find_filter (cache, symbol); | ||||
if (item) { | if (item) { | ||||
if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { | |||||
dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | |||||
dyn_item->finished = 0; | |||||
dyn_item->started = 0; | |||||
msg_debug_cache_task ("enable execution of %s", symbol); | |||||
} | |||||
else { | |||||
msg_debug_cache_task ("skip enabling squeezed symbol %s", symbol); | |||||
} | |||||
dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); | |||||
dyn_item->finished = 0; | |||||
dyn_item->started = 0; | |||||
msg_debug_cache_task ("enable execution of %s", symbol); | |||||
} | } | ||||
else { | else { | ||||
msg_info_task ("cannot enable %s: not found", symbol); | msg_info_task ("cannot enable %s: not found", symbol); | ||||
rspamd_task_profile_set (task, item->symbol, diff); | rspamd_task_profile_set (task, item->symbol, diff); | ||||
} | } | ||||
if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { | |||||
if (diff > slow_diff_limit) { | |||||
msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, | |||||
diff); | |||||
} | |||||
if (diff > slow_diff_limit) { | |||||
msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, | |||||
diff); | |||||
} | |||||
if (rspamd_worker_is_scanner (task->worker)) { | |||||
rspamd_set_counter (item->cd, diff); | |||||
} | |||||
if (rspamd_worker_is_scanner (task->worker)) { | |||||
rspamd_set_counter (item->cd, diff); | |||||
} | } | ||||
/* Process all reverse dependencies */ | /* Process all reverse dependencies */ |
SYMBOL_TYPE_POSTFILTER = (1u << 10u), | SYMBOL_TYPE_POSTFILTER = (1u << 10u), | ||||
SYMBOL_TYPE_NOSTAT = (1u << 11u), /* Skip as statistical symbol */ | SYMBOL_TYPE_NOSTAT = (1u << 11u), /* Skip as statistical symbol */ | ||||
SYMBOL_TYPE_IDEMPOTENT = (1u << 12u), /* Symbol cannot change metric */ | SYMBOL_TYPE_IDEMPOTENT = (1u << 12u), /* Symbol cannot change metric */ | ||||
SYMBOL_TYPE_SQUEEZED = (1u << 13u), /* Symbol is squeezed inside Lua */ | |||||
SYMBOL_TYPE_TRIVIAL = (1u << 14u), /* Symbol is trivial */ | SYMBOL_TYPE_TRIVIAL = (1u << 14u), /* Symbol is trivial */ | ||||
SYMBOL_TYPE_MIME_ONLY = (1u << 15u), /* Symbol is mime only */ | SYMBOL_TYPE_MIME_ONLY = (1u << 15u), /* Symbol is mime only */ | ||||
SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u), /* Symbol should be disabled explicitly only */ | SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u), /* Symbol should be disabled explicitly only */ |
rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol"); | rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol"); | ||||
} | } | ||||
static gint | |||||
rspamd_lua_squeeze_rule (lua_State *L, | |||||
struct rspamd_config *cfg, | |||||
const gchar *name, | |||||
gint cbref, | |||||
enum rspamd_symbol_type type, | |||||
gint parent) | |||||
{ | |||||
gint ret = -1, err_idx; | |||||
lua_pushcfunction (L, &rspamd_lua_traceback); | |||||
err_idx = lua_gettop (L); | |||||
if (type & SYMBOL_TYPE_VIRTUAL) { | |||||
if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_virtual")) { | |||||
lua_pushnumber (L, parent); | |||||
if (name) { | |||||
lua_pushstring (L, name); | |||||
} | |||||
else { | |||||
lua_pushnil (L); | |||||
} | |||||
/* Now call for squeeze function */ | |||||
if (lua_pcall (L, 2, 1, err_idx) != 0) { | |||||
msg_err_config ("call to squeeze_virtual failed: %s", | |||||
lua_tostring (L, -1)); | |||||
} | |||||
ret = lua_tonumber (L, -1); | |||||
} | |||||
else { | |||||
msg_err_config ("lua_squeeze is absent or bad (missing squeeze_virtual)," | |||||
" your Rspamd installation" | |||||
" is likely corrupted!"); | |||||
} | |||||
} | |||||
else if (type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) { | |||||
if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_rule")) { | |||||
if (name) { | |||||
lua_pushstring (L, name); | |||||
} | |||||
else { | |||||
lua_pushnil (L); | |||||
} | |||||
/* Push function reference */ | |||||
lua_rawgeti (L, LUA_REGISTRYINDEX, cbref); | |||||
/* Flags */ | |||||
lua_createtable (L, 0, 0); | |||||
if (type & SYMBOL_TYPE_MIME_ONLY) { | |||||
lua_pushstring (L, "mime"); | |||||
lua_pushboolean (L, true); | |||||
lua_settable (L, -3); | |||||
} | |||||
if (type & SYMBOL_TYPE_FINE) { | |||||
lua_pushstring (L, "fine"); | |||||
lua_pushboolean (L, true); | |||||
lua_settable (L, -3); | |||||
} | |||||
if (type & SYMBOL_TYPE_NOSTAT) { | |||||
lua_pushstring (L, "nostat"); | |||||
lua_pushboolean (L, true); | |||||
lua_settable (L, -3); | |||||
} | |||||
if (type & SYMBOL_TYPE_EXPLICIT_DISABLE) { | |||||
lua_pushstring (L, "explicit_disable"); | |||||
lua_pushboolean (L, true); | |||||
lua_settable (L, -3); | |||||
} | |||||
/* Now call for squeeze function */ | |||||
if (lua_pcall (L, 3, 1, err_idx) != 0) { | |||||
msg_err_config ("call to squeeze_rule failed: %s", | |||||
lua_tostring (L, -1)); | |||||
} | |||||
ret = lua_tonumber (L, -1); | |||||
} | |||||
else { | |||||
msg_err_config ("lua_squeeze is absent or bad (missing squeeze_rule)," | |||||
" your Rspamd installation" | |||||
" is likely corrupted!"); | |||||
} | |||||
} | |||||
/* No squeeze for everything else */ | |||||
/* Cleanup lua stack */ | |||||
lua_settop (L, err_idx - 1); | |||||
return ret; | |||||
} | |||||
static gint | static gint | ||||
rspamd_register_symbol_fromlua (lua_State *L, | rspamd_register_symbol_fromlua (lua_State *L, | ||||
struct rspamd_config *cfg, | struct rspamd_config *cfg, | ||||
gint priority, | gint priority, | ||||
enum rspamd_symbol_type type, | enum rspamd_symbol_type type, | ||||
gint parent, | gint parent, | ||||
gboolean optional, | |||||
gboolean no_squeeze) | |||||
gboolean optional) | |||||
{ | { | ||||
struct lua_callback_data *cd; | struct lua_callback_data *cd; | ||||
gint ret = -1; | gint ret = -1; | ||||
} | } | ||||
if (ref != -1) { | if (ref != -1) { | ||||
if (type & SYMBOL_TYPE_USE_CORO) { | |||||
/* Coroutines are incompatible with squeezing */ | |||||
no_squeeze = TRUE; | |||||
cd = rspamd_mempool_alloc0 (cfg->cfg_pool, | |||||
sizeof (struct lua_callback_data)); | |||||
cd->magic = rspamd_lua_callback_magic; | |||||
cd->cb_is_ref = TRUE; | |||||
cd->callback.ref = ref; | |||||
cd->L = L; | |||||
if (name) { | |||||
cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); | |||||
} | } | ||||
/* | |||||
* We call for routine called lua_squeeze_rules.squeeze_rule if it exists | |||||
*/ | |||||
if (no_squeeze || (ret = rspamd_lua_squeeze_rule (L, cfg, name, ref, | |||||
type, parent)) == -1) { | |||||
cd = rspamd_mempool_alloc0 (cfg->cfg_pool, | |||||
sizeof (struct lua_callback_data)); | |||||
cd->magic = rspamd_lua_callback_magic; | |||||
cd->cb_is_ref = TRUE; | |||||
cd->callback.ref = ref; | |||||
cd->L = L; | |||||
if (name) { | |||||
cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); | |||||
} | |||||
if (type & SYMBOL_TYPE_USE_CORO) { | |||||
ret = rspamd_symcache_add_symbol (cfg->cache, | |||||
name, | |||||
priority, | |||||
lua_metric_symbol_callback_coro, | |||||
cd, | |||||
type, | |||||
parent); | |||||
} | |||||
else { | |||||
ret = rspamd_symcache_add_symbol (cfg->cache, | |||||
name, | |||||
priority, | |||||
lua_metric_symbol_callback, | |||||
cd, | |||||
type, | |||||
parent); | |||||
} | |||||
rspamd_mempool_add_destructor (cfg->cfg_pool, | |||||
(rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, | |||||
cd); | |||||
if (type & SYMBOL_TYPE_USE_CORO) { | |||||
ret = rspamd_symcache_add_symbol (cfg->cache, | |||||
name, | |||||
priority, | |||||
lua_metric_symbol_callback_coro, | |||||
cd, | |||||
type, | |||||
parent); | |||||
} | |||||
else { | |||||
ret = rspamd_symcache_add_symbol (cfg->cache, | |||||
name, | |||||
priority, | |||||
lua_metric_symbol_callback, | |||||
cd, | |||||
type, | |||||
parent); | |||||
} | } | ||||
rspamd_mempool_add_destructor (cfg->cfg_pool, | |||||
(rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, | |||||
cd); | |||||
} | } | ||||
else { | else { | ||||
if (!no_squeeze) { | |||||
rspamd_lua_squeeze_rule (L, cfg, name, ref, | |||||
type, parent); | |||||
} | |||||
/* Not a squeezed symbol */ | |||||
/* No callback */ | |||||
ret = rspamd_symcache_add_symbol (cfg->cache, | ret = rspamd_symcache_add_symbol (cfg->cache, | ||||
name, | name, | ||||
priority, | priority, | ||||
order, | order, | ||||
SYMBOL_TYPE_POSTFILTER|SYMBOL_TYPE_CALLBACK, | SYMBOL_TYPE_POSTFILTER|SYMBOL_TYPE_CALLBACK, | ||||
-1, | -1, | ||||
FALSE, | |||||
TRUE); | |||||
FALSE); | |||||
lua_pushboolean (L, ret); | lua_pushboolean (L, ret); | ||||
} | } | ||||
order, | order, | ||||
SYMBOL_TYPE_PREFILTER|SYMBOL_TYPE_CALLBACK, | SYMBOL_TYPE_PREFILTER|SYMBOL_TYPE_CALLBACK, | ||||
-1, | -1, | ||||
FALSE, | |||||
TRUE); | |||||
FALSE); | |||||
lua_pushboolean (L, ret); | lua_pushboolean (L, ret); | ||||
} | } | ||||
if (strstr (str, "idempotent") != NULL) { | if (strstr (str, "idempotent") != NULL) { | ||||
ret |= SYMBOL_TYPE_IDEMPOTENT; | ret |= SYMBOL_TYPE_IDEMPOTENT; | ||||
} | } | ||||
if (strstr (str, "squeezed") != NULL) { | |||||
ret |= SYMBOL_TYPE_SQUEEZED; | |||||
} | |||||
if (strstr (str, "trivial") != NULL) { | if (strstr (str, "trivial") != NULL) { | ||||
ret |= SYMBOL_TYPE_TRIVIAL; | ret |= SYMBOL_TYPE_TRIVIAL; | ||||
} | } | ||||
lua_rawseti (L, -2, i++); | lua_rawseti (L, -2, i++); | ||||
} | } | ||||
if (flags & SYMBOL_TYPE_SQUEEZED) { | |||||
lua_pushstring (L, "squeezed"); | |||||
lua_rawseti (L, -2, i++); | |||||
} | |||||
if (flags & SYMBOL_TYPE_EXPLICIT_DISABLE) { | if (flags & SYMBOL_TYPE_EXPLICIT_DISABLE) { | ||||
lua_pushstring (L, "explicit_disable"); | lua_pushstring (L, "explicit_disable"); | ||||
lua_rawseti (L, -2, i++); | lua_rawseti (L, -2, i++); | ||||
const gchar *name = NULL, *flags_str = NULL, *type_str = NULL, | const gchar *name = NULL, *flags_str = NULL, *type_str = NULL, | ||||
*description = NULL, *group = NULL; | *description = NULL, *group = NULL; | ||||
double weight = 0, score = NAN, parent_float = NAN; | double weight = 0, score = NAN, parent_float = NAN; | ||||
gboolean one_shot = FALSE, no_squeeze = FALSE; | |||||
gboolean one_shot = FALSE; | |||||
gint ret = -1, cbref = -1, type, flags = 0; | gint ret = -1, cbref = -1, type, flags = 0; | ||||
gint64 parent = 0, priority = 0, nshots = 0; | gint64 parent = 0, priority = 0, nshots = 0; | ||||
GError *err = NULL; | GError *err = NULL; | ||||
if (cfg) { | if (cfg) { | ||||
if (!rspamd_lua_parse_table_arguments (L, 2, &err, | if (!rspamd_lua_parse_table_arguments (L, 2, &err, | ||||
"name=S;weight=N;callback=F;flags=S;type=S;priority=I;parent=D;" | "name=S;weight=N;callback=F;flags=S;type=S;priority=I;parent=D;" | ||||
"score=D;description=S;group=S;one_shot=B;nshots=I;no_squeeze=B", | |||||
"score=D;description=S;group=S;one_shot=B;nshots=I", | |||||
&name, &weight, &cbref, &flags_str, &type_str, | &name, &weight, &cbref, &flags_str, &type_str, | ||||
&priority, &parent_float, | &priority, &parent_float, | ||||
&score, &description, &group, &one_shot, &nshots, &no_squeeze)) { | |||||
&score, &description, &group, &one_shot, &nshots)) { | |||||
msg_err_config ("bad arguments: %e", err); | msg_err_config ("bad arguments: %e", err); | ||||
g_error_free (err); | g_error_free (err); | ||||
return luaL_error (L, "invalid arguments"); | return luaL_error (L, "invalid arguments"); | ||||
} | } | ||||
if (!no_squeeze) { | |||||
no_squeeze = cfg->disable_lua_squeeze; | |||||
} | |||||
if (nshots == 0) { | if (nshots == 0) { | ||||
nshots = cfg->default_max_shots; | nshots = cfg->default_max_shots; | ||||
} | } | ||||
} | } | ||||
if (flags_str) { | if (flags_str) { | ||||
/* Turn off squeezing as well for now */ | |||||
/* TODO: deal with it */ | |||||
no_squeeze = TRUE; | |||||
type |= lua_parse_symbol_flags (flags_str); | type |= lua_parse_symbol_flags (flags_str); | ||||
} | } | ||||
priority, | priority, | ||||
type, | type, | ||||
parent, | parent, | ||||
FALSE, | |||||
no_squeeze); | |||||
FALSE); | |||||
if (!isnan (score) || group) { | if (!isnan (score) || group) { | ||||
if (one_shot) { | if (one_shot) { | ||||
0, | 0, | ||||
SYMBOL_TYPE_CALLBACK, | SYMBOL_TYPE_CALLBACK, | ||||
-1, | -1, | ||||
FALSE, | |||||
FALSE); | FALSE); | ||||
for (i = top; i <= lua_gettop (L); i++) { | for (i = top; i <= lua_gettop (L); i++) { | ||||
0, | 0, | ||||
SYMBOL_TYPE_CALLBACK, | SYMBOL_TYPE_CALLBACK, | ||||
-1, | -1, | ||||
FALSE, | |||||
lua_type (L, top + 1) == LUA_TSTRING); | |||||
FALSE); | |||||
} | } | ||||
lua_pushinteger (L, ret); | lua_pushinteger (L, ret); | ||||
priority, | priority, | ||||
SYMBOL_TYPE_CALLBACK, | SYMBOL_TYPE_CALLBACK, | ||||
-1, | -1, | ||||
FALSE, | |||||
lua_type (L, top + 2) == LUA_TSTRING); | |||||
FALSE); | |||||
} | } | ||||
lua_pushinteger (L, ret); | lua_pushinteger (L, ret); | ||||
return 1; | return 1; | ||||
} | } | ||||
static gboolean | |||||
rspamd_lua_squeeze_dependency (lua_State *L, struct rspamd_config *cfg, | |||||
const gchar *child, | |||||
const gchar *parent) | |||||
{ | |||||
gint err_idx; | |||||
gboolean ret = FALSE; | |||||
g_assert (parent != NULL); | |||||
g_assert (child != NULL); | |||||
lua_pushcfunction (L, &rspamd_lua_traceback); | |||||
err_idx = lua_gettop (L); | |||||
if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_dependency")) { | |||||
lua_pushstring (L, child); | |||||
lua_pushstring (L, parent); | |||||
if (lua_pcall (L, 2, 1, err_idx) != 0) { | |||||
msg_err_config ("call to squeeze_dependency script failed: %s", | |||||
lua_tostring (L, -1)); | |||||
} | |||||
else { | |||||
ret = lua_toboolean (L, -1); | |||||
} | |||||
} | |||||
else { | |||||
msg_err_config ("cannot get lua_squeeze_rules.squeeze_dependency function"); | |||||
} | |||||
lua_settop (L, err_idx - 1); | |||||
return ret; | |||||
} | |||||
static gint | static gint | ||||
lua_config_register_dependency (lua_State * L) | lua_config_register_dependency (lua_State * L) | ||||
struct rspamd_config *cfg = lua_check_config (L, 1); | struct rspamd_config *cfg = lua_check_config (L, 1); | ||||
const gchar *parent = NULL, *child = NULL; | const gchar *parent = NULL, *child = NULL; | ||||
gint child_id; | gint child_id; | ||||
gboolean skip_squeeze; | |||||
if (cfg == NULL) { | if (cfg == NULL) { | ||||
lua_error (L); | lua_error (L); | ||||
return 0; | return 0; | ||||
} | } | ||||
skip_squeeze = cfg->disable_lua_squeeze; | |||||
if (lua_type (L, 2) == LUA_TNUMBER) { | if (lua_type (L, 2) == LUA_TNUMBER) { | ||||
child_id = luaL_checknumber (L, 2); | child_id = luaL_checknumber (L, 2); | ||||
parent = luaL_checkstring (L, 3); | parent = luaL_checkstring (L, 3); | ||||
if (!skip_squeeze && lua_isboolean (L, 4)) { | |||||
skip_squeeze = lua_toboolean (L, 4); | |||||
} | |||||
msg_warn_config ("calling for obsolete method to register deps for symbol %d->%s", | msg_warn_config ("calling for obsolete method to register deps for symbol %d->%s", | ||||
child_id, parent); | child_id, parent); | ||||
if (child_id > 0 && parent != NULL) { | if (child_id > 0 && parent != NULL) { | ||||
if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, | |||||
rspamd_symcache_symbol_by_id (cfg->cache, child_id), | |||||
parent)) { | |||||
rspamd_symcache_add_dependency (cfg->cache, child_id, parent); | |||||
} | |||||
rspamd_symcache_add_dependency (cfg->cache, child_id, parent); | |||||
} | } | ||||
} | } | ||||
else { | else { | ||||
child = luaL_checkstring (L,2); | child = luaL_checkstring (L,2); | ||||
parent = luaL_checkstring (L, 3); | parent = luaL_checkstring (L, 3); | ||||
if (!skip_squeeze && lua_isboolean (L, 4)) { | |||||
skip_squeeze = lua_toboolean (L, 4); | |||||
} | |||||
if (child != NULL && parent != NULL) { | if (child != NULL && parent != NULL) { | ||||
if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, child, parent)) { | |||||
rspamd_symcache_add_delayed_dependency (cfg->cache, child, | |||||
parent); | |||||
} | |||||
rspamd_symcache_add_delayed_dependency (cfg->cache, child, | |||||
parent); | |||||
} | } | ||||
} | } | ||||
struct rspamd_config *cfg = lua_check_config (L, 1); | struct rspamd_config *cfg = lua_check_config (L, 1); | ||||
const gchar *name; | const gchar *name; | ||||
gint id, nshots, flags = 0; | gint id, nshots, flags = 0; | ||||
gboolean optional = FALSE, no_squeeze = FALSE; | |||||
gboolean optional = FALSE; | |||||
name = luaL_checkstring (L, 2); | name = luaL_checkstring (L, 2); | ||||
if (cfg != NULL && name != NULL && lua_gettop (L) == 3) { | if (cfg != NULL && name != NULL && lua_gettop (L) == 3) { | ||||
no_squeeze = cfg->disable_lua_squeeze; | |||||
if (lua_type (L, 3) == LUA_TFUNCTION) { | if (lua_type (L, 3) == LUA_TFUNCTION) { | ||||
/* Normal symbol from just a function */ | /* Normal symbol from just a function */ | ||||
0, | 0, | ||||
SYMBOL_TYPE_NORMAL, | SYMBOL_TYPE_NORMAL, | ||||
-1, | -1, | ||||
FALSE, | |||||
no_squeeze); | |||||
FALSE); | |||||
} | } | ||||
else if (lua_type (L, 3) == LUA_TTABLE) { | else if (lua_type (L, 3) == LUA_TTABLE) { | ||||
gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx; | gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx; | ||||
gdouble weight = 1.0, score = NAN; | gdouble weight = 1.0, score = NAN; | ||||
const char *type_str, *group = NULL, *description = NULL; | const char *type_str, *group = NULL, *description = NULL; | ||||
no_squeeze = cfg->disable_lua_squeeze; | |||||
/* | /* | ||||
* Table can have the following attributes: | * Table can have the following attributes: | ||||
* "callback" - should be a callback function | * "callback" - should be a callback function | ||||
} | } | ||||
lua_pop (L, 1); | lua_pop (L, 1); | ||||
lua_pushstring (L, "condition"); | |||||
lua_gettable (L, -2); | |||||
if (lua_type (L, -1) == LUA_TFUNCTION) { | |||||
no_squeeze = TRUE; | |||||
} | |||||
lua_pop (L, 1); | |||||
if (!no_squeeze) { | |||||
lua_pushstring (L, "no_squeeze"); | |||||
lua_gettable (L, -2); | |||||
if (lua_toboolean (L, -1)) { | |||||
no_squeeze = TRUE; | |||||
} | |||||
lua_pop (L, 1); | |||||
} | |||||
id = rspamd_register_symbol_fromlua (L, | id = rspamd_register_symbol_fromlua (L, | ||||
cfg, | cfg, | ||||
name, | name, | ||||
priority, | priority, | ||||
type, | type, | ||||
-1, | -1, | ||||
optional, | |||||
no_squeeze); | |||||
optional); | |||||
if (id != -1) { | if (id != -1) { | ||||
/* Check for condition */ | /* Check for condition */ |
local rspamd_logger = require "rspamd_logger" | local rspamd_logger = require "rspamd_logger" | ||||
local rspamd_maps = require "lua_maps" | local rspamd_maps = require "lua_maps" | ||||
local lua_squeeze = require "lua_squeeze_rules" | |||||
local lua_util = require "lua_util" | local lua_util = require "lua_util" | ||||
local rspamd_util = require "rspamd_util" | local rspamd_util = require "rspamd_util" | ||||
local rspamd_ip = require "rspamd_ip" | local rspamd_ip = require "rspamd_ip" | ||||
local function apply_settings(task, to_apply) | local function apply_settings(task, to_apply) | ||||
task:set_settings(to_apply) | task:set_settings(to_apply) | ||||
task:cache_set('settings', to_apply) | task:cache_set('settings', to_apply) | ||||
lua_squeeze.handle_settings(task, to_apply) | |||||
if to_apply['add_headers'] or to_apply['remove_headers'] then | if to_apply['add_headers'] or to_apply['remove_headers'] then | ||||
local rep = { | local rep = { |