From 615df8d1fa4ad75dc14fc29e36ac18b118df50b1 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 10 Jun 2019 16:33:19 +0100 Subject: [PATCH] [Rework] Eliminate lua_squeeze as it has shown no improvements --- lualib/lua_squeeze_rules.lua | 422 -------------------------------- src/libserver/cfg_file.h | 1 - src/libserver/cfg_rcl.c | 6 - src/libserver/cfg_utils.c | 17 -- src/libserver/rspamd_symcache.c | 68 +++-- src/libserver/rspamd_symcache.h | 1 - src/lua/lua_config.c | 296 ++++------------------ src/plugins/lua/settings.lua | 2 - 8 files changed, 78 insertions(+), 735 deletions(-) delete mode 100644 lualib/lua_squeeze_rules.lua diff --git a/lualib/lua_squeeze_rules.lua b/lualib/lua_squeeze_rules.lua deleted file mode 100644 index 309d19969..000000000 --- a/lualib/lua_squeeze_rules.lua +++ /dev/null @@ -1,422 +0,0 @@ ---[[ -Copyright (c) 2018, Vsevolod Stakhov - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -]]-- - -local exports = {} -local logger = require 'rspamd_logger' -local lua_util = require 'lua_util' - --- Squeezed rules part -local squeezed_rules = {{}} -- plain vector of all rules squeezed -local squeezed_symbols = {} -- indexed by name of symbol -local squeezed_deps = {} -- squeezed deps -local squeezed_rdeps = {} -- squeezed reverse deps -local SN = 'lua_squeeze' -local squeeze_sym = 'LUA_SQUEEZE' -local squeeze_function_ids = {} -local squeezed_groups = {} -local last_rule -local virtual_symbols = {} - -local function gen_lua_squeeze_function(order) - return function(task) - local symbols_disabled = task:cache_get('squeezed_disable') - local mime_task = task:has_flag('mime') - for _,data in ipairs(squeezed_rules[order]) do - local disable = false - if symbols_disabled and symbols_disabled[data[2]] then - disable = true - end - - if data[3] and data[3].flags.mime then - if not mime_task then - disable = true - end - end - - if not disable then - local function real_call() - return {data[1](task)} - end - - -- Too expensive to call :( - lua_util.debugm(SN, task, 'call for: %s', data[2]) - local status, ret = pcall(real_call) - - if not status then - logger.errx(task, 'error in squeezed rule %s: %s', data[2], ret) - else - if #ret ~= 0 then - local first = ret[1] - local sym = data[2] - -- Function has returned something, so it is rule, not a plugin - if type(first) == 'boolean' then - if first then - table.remove(ret, 1) - - local second = ret[1] - - if type(second) == 'number' then - table.remove(ret, 1) - if second ~= 0 then - if type(ret[1]) == 'table' then - task:insert_result(sym, second, ret[1]) - else - task:insert_result(sym, second, ret) - end - end - else - if type(ret[1]) == 'table' then - task:insert_result(sym, 1.0, ret[1]) - else - task:insert_result(sym, 1.0, ret) - end - end - end - elseif type(first) == 'number' then - table.remove(ret, 1) - - if first ~= 0 then - if type(ret[1]) == 'table' then - task:insert_result(sym, first, ret[1]) - else - task:insert_result(sym, first, ret) - end - end - else - if type(ret[1]) == 'table' then - task:insert_result(sym, 1.0, ret[1]) - else - task:insert_result(sym, 1.0, ret) - end - end - end - end - else - lua_util.debugm(SN, task, 'skip symbol due to settings: %s', data[2]) - end - - - end - end -end - -exports.squeeze_rule = function(s, func, flags) - if s then - if not squeezed_symbols[s] then - squeezed_symbols[s] = { - cb = func, - order = 0, - sym = s, - flags = flags or {} - } - lua_util.debugm(SN, rspamd_config, 'squeezed rule: %s', s) - else - logger.warnx(rspamd_config, 'duplicate symbol registered: %s, skip', s) - end - else - -- Unconditionally add function to the squeezed rules - local id = tostring(#squeezed_rules) - lua_util.debugm(SN, rspamd_config, 'squeezed unnamed rule: %s', id) - table.insert(squeezed_rules[1], {func, 'unnamed: ' .. id, squeezed_symbols[s]}) - end - - if not squeeze_function_ids[1] then - squeeze_function_ids[1] = rspamd_config:register_symbol{ - type = 'callback', - flags = 'squeezed', - callback = gen_lua_squeeze_function(1), - name = squeeze_sym, - description = 'Meta rule for Lua rules that can be squeezed', - no_squeeze = true, -- to avoid infinite recursion - } - end - - last_rule = s - - return squeeze_function_ids[1] -end - --- TODO: poor approach, we register all virtual symbols with the previous real squeezed symbol -exports.squeeze_virtual = function(id, name) - - if squeeze_function_ids[1] and id == squeeze_function_ids[1] and last_rule then - virtual_symbols[name] = last_rule - lua_util.debugm(SN, rspamd_config, 'add virtual symbol %s -> %s', - name, last_rule) - - return id - end - - return -1 -end - -exports.squeeze_dependency = function(child, parent) - lua_util.debugm(SN, rspamd_config, 'squeeze dep %s->%s', child, parent) - - if not squeezed_deps[parent] then - squeezed_deps[parent] = {} - end - - if not squeezed_deps[parent][child] then - squeezed_deps[parent][child] = true - else - logger.warnx(rspamd_config, 'duplicate dependency %s->%s', child, parent) - end - - if not squeezed_rdeps[child] then - squeezed_rdeps[child] = {} - end - - if not squeezed_rdeps[child][parent] then - squeezed_rdeps[child][parent] = true - end - - return true -end - -local function get_ordered_symbol_name(order) - if order == 1 then - return squeeze_sym - end - - return squeeze_sym .. tostring(order) -end - -local function register_topology_symbol(order) - local ord_sym = get_ordered_symbol_name(order) - - squeeze_function_ids[order] = rspamd_config:register_symbol{ - type = 'callback', - flags = 'squeezed', - callback = gen_lua_squeeze_function(order), - name = ord_sym, - description = 'Meta rule for Lua rules that can be squeezed, order ' .. tostring(order), - no_squeeze = true, -- to avoid infinite recursion - } - - local parent = get_ordered_symbol_name(order - 1) - lua_util.debugm(SN, rspamd_config, 'registered new order of deps: %s->%s', - ord_sym, parent) - rspamd_config:register_dependency(ord_sym, parent, true) -end - -exports.squeeze_init = function() - -- Do topological sorting - for _,v in pairs(squeezed_symbols) do - local function visit(node, order) - - if order > node.order then - node.order = order - lua_util.debugm(SN, rspamd_config, "symbol: %s, order: %s", node.sym, order) - else - return - end - - if squeezed_deps[node.sym] then - for dep,_ in pairs(squeezed_deps[node.sym]) do - if squeezed_symbols[dep] then - visit(squeezed_symbols[dep], order + 1) - end - end - end - end - - if v.order == 0 then - visit(v, 1) - end - end - - for parent,children in pairs(squeezed_deps) do - if not squeezed_symbols[parent] then - local real_parent = virtual_symbols[parent] - if real_parent then - parent = real_parent - end - end - - if not squeezed_symbols[parent] then - -- Trivial case, external dependnency - - for s,_ in pairs(children) do - - if squeezed_symbols[s] then - -- External dep depends on a squeezed symbol - lua_util.debugm(SN, rspamd_config, 'register external squeezed dependency on %s', - parent) - rspamd_config:register_dependency(squeeze_sym, parent, true) - else - -- Generic rspamd symbols dependency - lua_util.debugm(SN, rspamd_config, 'register external dependency %s -> %s', - s, parent) - rspamd_config:register_dependency(s, parent, true) - end - end - else - -- Not so trivial case - local ps = squeezed_symbols[parent] - - for cld,_ in pairs(children) do - if squeezed_symbols[cld] then - -- Cross dependency - lua_util.debugm(SN, rspamd_config, 'cross dependency in squeezed symbols %s->%s', - cld, parent) - local order = squeezed_symbols[cld].order - if not squeeze_function_ids[order] then - -- Need to register new callback symbol to handle deps - for i = 1, order do - if not squeeze_function_ids[i] then - register_topology_symbol(i) - end - end - end - else - -- External symbol depends on a squeezed one - local parent_symbol = get_ordered_symbol_name(ps.order) - rspamd_config:register_dependency(cld, parent_symbol, true) - lua_util.debugm(SN, rspamd_config, 'register squeezed dependency for external symbol %s->%s', - cld, parent_symbol) - end - end - end - end - - -- We have now all deps being registered, so we can register virtual symbols - -- and create squeezed rules - for k,v in pairs(squeezed_symbols) do - local parent_symbol = get_ordered_symbol_name(v.order) - lua_util.debugm(SN, rspamd_config, 'added squeezed rule: %s (%s): %s', - k, parent_symbol, v) - rspamd_config:register_symbol{ - type = 'virtual', - name = k, - flags = 'squeezed', - parent = squeeze_function_ids[v.order], - no_squeeze = true, -- to avoid infinite recursion - } - local metric_sym = rspamd_config:get_metric_symbol(k) - - if metric_sym then - v.group = metric_sym.group - v.groups = metric_sym.groups - v.score = metric_sym.score - v.description = metric_sym.description - - if v.group then - if not squeezed_groups[v.group] then - lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', v.group) - squeezed_groups[v.group] = {} - end - - table.insert(squeezed_groups[v.group], k) - - end - if v.groups then - for _,gr in ipairs(v.groups) do - if not squeezed_groups[gr] then - lua_util.debugm(SN, rspamd_config, 'added squeezed group: %s', gr) - squeezed_groups[gr] = {} - end - - table.insert(squeezed_groups[gr], k) - end - end - else - lua_util.debugm(SN, rspamd_config, 'no metric symbol found for %s, maybe bug', k) - end - if not squeezed_rules[v.order] then - squeezed_rules[v.order] = {} - end - table.insert(squeezed_rules[v.order], {v.cb,k,v}) - end -end - -exports.handle_settings = function(task, settings) - local symbols_disabled = {} - local symbols_enabled = {} - local found = false - local disabled = false - - if settings.default then settings = settings.default end - - local function disable_all() - for k,sym in pairs(squeezed_symbols) do - if not symbols_enabled[k] and not (sym.flags and sym.flags.explicit_disable) then - symbols_disabled[k] = true - end - end - end - - if settings.symbols_enabled then - disable_all() - found = true - disabled = true - for _,s in ipairs(settings.symbols_enabled) do - if squeezed_symbols[s] then - lua_util.debugm(SN, task, 'enable symbol %s as it is in `symbols_enabled`', s) - symbols_enabled[s] = true - symbols_disabled[s] = nil - end - end - end - - if settings.groups_enabled then - if not disabled then - disable_all() - end - found = true - for _,gr in ipairs(settings.groups_enabled) do - if squeezed_groups[gr] then - for _,sym in ipairs(squeezed_groups[gr]) do - lua_util.debugm(SN, task, 'enable symbol %s as it is in `groups_enabled`', sym) - symbols_enabled[sym] = true - symbols_disabled[sym] = nil - end - end - end - end - - if settings.symbols_disabled then - found = true - for _,s in ipairs(settings.symbols_disabled) do - lua_util.debugm(SN, task, 'try disable symbol %s as it is in `symbols_disabled`', s) - if not symbols_enabled[s] then - symbols_disabled[s] = true - lua_util.debugm(SN, task, 'disable symbol %s as it is in `symbols_disabled`', s) - end - end - end - - if settings.groups_disabled then - found = true - for _,gr in ipairs(settings.groups_disabled) do - lua_util.debugm(SN, task, 'try disable group %s as it is in `groups_disabled`: %s', gr) - if squeezed_groups[gr] then - for _,sym in ipairs(squeezed_groups[gr]) do - if not symbols_enabled[sym] then - lua_util.debugm(SN, task, 'disable symbol %s as it is in `groups_disabled`', sym) - symbols_disabled[sym] = true - end - end - end - end - end - - if found then - task:cache_set('squeezed_disable', symbols_disabled) - end -end - -return exports \ No newline at end of file diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index eccab1217..b5780a599 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -336,7 +336,6 @@ struct rspamd_config { gboolean enable_sessions_cache; /**< Enable session cache for debug */ gboolean enable_experimental; /**< Enable experimental plugins */ gboolean disable_pcre_jit; /**< Disable pcre JIT */ - gboolean disable_lua_squeeze; /**< Disable lua rules squeezing */ gboolean own_lua_state; /**< True if we have created lua_state internally */ gboolean soft_reject_on_timeout; /**< If true emit soft reject on task timeout (if not reject) */ diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index 4c069db9e..db1b54cf8 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -1923,12 +1923,6 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections) G_STRUCT_OFFSET (struct rspamd_config, disable_pcre_jit), 0, "Disable PCRE JIT"); - rspamd_rcl_add_default_handler (sub, - "disable_lua_squeeze", - rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, disable_lua_squeeze), - 0, - "Disable Lua rules squeezing"); rspamd_rcl_add_default_handler (sub, "min_word_len", rspamd_rcl_parse_struct_integer, diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 586625184..1c8b2ce5f 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -819,23 +819,6 @@ rspamd_config_post_load (struct rspamd_config *cfg, } if (opts & RSPAMD_CONFIG_INIT_SYMCACHE) { - lua_State *L = cfg->lua_state; - int err_idx; - - /* Process squeezed Lua rules */ - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); - - if (rspamd_lua_require_function (cfg->lua_state, "lua_squeeze_rules", - "squeeze_init")) { - if (lua_pcall (L, 0, 0, err_idx) != 0) { - msg_err_config ("call to squeeze_init script failed: %s", - lua_tostring (L, -1)); - } - } - - lua_settop (L, err_idx - 1); - /* Init config cache */ rspamd_symcache_init (cfg->cache); diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c index bc3b5295c..42e40b68c 100644 --- a/src/libserver/rspamd_symcache.c +++ b/src/libserver/rspamd_symcache.c @@ -66,7 +66,6 @@ INIT_LOG_MODULE(symcache) (dyn_item)->finished = 1 #define CLR_FINISH_BIT(checkpoint, dyn_item) \ (dyn_item)->finished = 0 - static const guchar rspamd_symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 }; struct rspamd_symcache_header { @@ -82,6 +81,21 @@ struct symcache_order { ref_entry_t ref; }; +/* + * This structure is optimised to store ids list: + * - If the first element is -1 then use dynamic part, else use static part + */ +struct rspamd_symcache_id_list { + union { + guint32 st[4]; + struct { + guint32 e; + guint32 dynlen; + guint *n; + } dyn; + }; +}; + struct rspamd_symcache_item { /* This block is likely shared */ struct item_stat *st; @@ -115,6 +129,9 @@ struct rspamd_symcache_item { guint order; gint id; gint frequency_peaks; + /* Settings ids */ + struct rspamd_symcache_id_list allowed_ids; + struct rspamd_symcache_id_list forbidden_ids; /* Dependencies */ GPtrArray *deps; @@ -143,7 +160,6 @@ struct rspamd_symcache { GPtrArray *composites; GPtrArray *idempotent; GPtrArray *virtual; - GPtrArray *squeezed; GList *delayed_deps; GList *delayed_conditions; rspamd_mempool_t *static_pool; @@ -984,10 +1000,6 @@ rspamd_symcache_add_symbol (struct rspamd_symcache *cache, } } - if (item->type & SYMBOL_TYPE_SQUEEZED) { - g_ptr_array_add (cache->squeezed, item); - } - cache->used_items ++; cache->id ++; @@ -1127,7 +1139,6 @@ rspamd_symcache_destroy (struct rspamd_symcache *cache) g_ptr_array_free (cache->postfilters, TRUE); g_ptr_array_free (cache->idempotent, TRUE); g_ptr_array_free (cache->composites, TRUE); - g_ptr_array_free (cache->squeezed, TRUE); REF_RELEASE (cache->items_by_order); if (cache->peak_cb != -1) { @@ -1155,7 +1166,6 @@ rspamd_symcache_new (struct rspamd_config *cfg) cache->idempotent = g_ptr_array_new (); cache->composites = g_ptr_array_new (); cache->virtual = g_ptr_array_new (); - cache->squeezed = g_ptr_array_new (); cache->reload_time = cfg->cache_reload_time; cache->total_hits = 1; cache->total_weight = 1.0; @@ -2331,7 +2341,7 @@ rspamd_symcache_disable_all_symbols (struct rspamd_task *task, PTR_ARRAY_FOREACH (cache->items_by_id, i, item) { dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - if (!(item->type & (SYMBOL_TYPE_SQUEEZED|skip_mask))) { + if (!(item->type & (skip_mask))) { SET_FINISH_BIT (checkpoint, dyn_item); SET_START_BIT (checkpoint, dyn_item); } @@ -2357,15 +2367,10 @@ rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task, item = rspamd_symcache_find_filter (cache, symbol); if (item) { - if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { - dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - SET_FINISH_BIT (checkpoint, dyn_item); - SET_START_BIT (checkpoint, dyn_item); - msg_debug_cache_task ("disable execution of %s", symbol); - } - else { - msg_debug_cache_task ("skip disabling squeezed symbol %s", symbol); - } + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + SET_FINISH_BIT (checkpoint, dyn_item); + SET_START_BIT (checkpoint, dyn_item); + msg_debug_cache_task ("disable execution of %s", symbol); } else { msg_info_task ("cannot disable %s: not found", symbol); @@ -2391,15 +2396,10 @@ rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task, item = rspamd_symcache_find_filter (cache, symbol); if (item) { - if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { - dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); - dyn_item->finished = 0; - dyn_item->started = 0; - msg_debug_cache_task ("enable execution of %s", symbol); - } - else { - msg_debug_cache_task ("skip enabling squeezed symbol %s", symbol); - } + dyn_item = rspamd_symcache_get_dynamic (checkpoint, item); + dyn_item->finished = 0; + dyn_item->started = 0; + msg_debug_cache_task ("enable execution of %s", symbol); } else { msg_info_task ("cannot enable %s: not found", symbol); @@ -2726,15 +2726,13 @@ rspamd_symcache_finalize_item (struct rspamd_task *task, rspamd_task_profile_set (task, item->symbol, diff); } - if (!(item->type & SYMBOL_TYPE_SQUEEZED)) { - if (diff > slow_diff_limit) { - msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, - diff); - } + if (diff > slow_diff_limit) { + msg_info_task ("slow rule: %s(%d): %.2f ms", item->symbol, item->id, + diff); + } - if (rspamd_worker_is_scanner (task->worker)) { - rspamd_set_counter (item->cd, diff); - } + if (rspamd_worker_is_scanner (task->worker)) { + rspamd_set_counter (item->cd, diff); } /* Process all reverse dependencies */ diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h index afdda7e79..728b7636c 100644 --- a/src/libserver/rspamd_symcache.h +++ b/src/libserver/rspamd_symcache.h @@ -45,7 +45,6 @@ enum rspamd_symbol_type { SYMBOL_TYPE_POSTFILTER = (1u << 10u), SYMBOL_TYPE_NOSTAT = (1u << 11u), /* Skip as statistical symbol */ SYMBOL_TYPE_IDEMPOTENT = (1u << 12u), /* Symbol cannot change metric */ - SYMBOL_TYPE_SQUEEZED = (1u << 13u), /* Symbol is squeezed inside Lua */ SYMBOL_TYPE_TRIVIAL = (1u << 14u), /* Symbol is trivial */ SYMBOL_TYPE_MIME_ONLY = (1u << 15u), /* Symbol is mime only */ SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u), /* Symbol should be disabled explicitly only */ diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index cc0fa44ee..f2e9abd01 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -1365,101 +1365,6 @@ lua_metric_symbol_callback_return (struct thread_entry *thread_entry, int ret) rspamd_symcache_item_async_dec_check (task, cd->item, "lua coro symbol"); } -static gint -rspamd_lua_squeeze_rule (lua_State *L, - struct rspamd_config *cfg, - const gchar *name, - gint cbref, - enum rspamd_symbol_type type, - gint parent) -{ - gint ret = -1, err_idx; - - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); - - if (type & SYMBOL_TYPE_VIRTUAL) { - if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_virtual")) { - lua_pushnumber (L, parent); - if (name) { - lua_pushstring (L, name); - } - else { - lua_pushnil (L); - } - - /* Now call for squeeze function */ - if (lua_pcall (L, 2, 1, err_idx) != 0) { - msg_err_config ("call to squeeze_virtual failed: %s", - lua_tostring (L, -1)); - } - - ret = lua_tonumber (L, -1); - } - else { - msg_err_config ("lua_squeeze is absent or bad (missing squeeze_virtual)," - " your Rspamd installation" - " is likely corrupted!"); - } - } - else if (type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) { - if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_rule")) { - if (name) { - lua_pushstring (L, name); - } - else { - lua_pushnil (L); - } - - /* Push function reference */ - lua_rawgeti (L, LUA_REGISTRYINDEX, cbref); - - /* Flags */ - lua_createtable (L, 0, 0); - - if (type & SYMBOL_TYPE_MIME_ONLY) { - lua_pushstring (L, "mime"); - lua_pushboolean (L, true); - lua_settable (L, -3); - } - if (type & SYMBOL_TYPE_FINE) { - lua_pushstring (L, "fine"); - lua_pushboolean (L, true); - lua_settable (L, -3); - } - if (type & SYMBOL_TYPE_NOSTAT) { - lua_pushstring (L, "nostat"); - lua_pushboolean (L, true); - lua_settable (L, -3); - } - if (type & SYMBOL_TYPE_EXPLICIT_DISABLE) { - lua_pushstring (L, "explicit_disable"); - lua_pushboolean (L, true); - lua_settable (L, -3); - } - - /* Now call for squeeze function */ - if (lua_pcall (L, 3, 1, err_idx) != 0) { - msg_err_config ("call to squeeze_rule failed: %s", - lua_tostring (L, -1)); - } - - ret = lua_tonumber (L, -1); - } - else { - msg_err_config ("lua_squeeze is absent or bad (missing squeeze_rule)," - " your Rspamd installation" - " is likely corrupted!"); - } - } - /* No squeeze for everything else */ - - /* Cleanup lua stack */ - lua_settop (L, err_idx - 1); - - return ret; -} - static gint rspamd_register_symbol_fromlua (lua_State *L, struct rspamd_config *cfg, @@ -1469,8 +1374,7 @@ rspamd_register_symbol_fromlua (lua_State *L, gint priority, enum rspamd_symbol_type type, gint parent, - gboolean optional, - gboolean no_squeeze) + gboolean optional) { struct lua_callback_data *cd; gint ret = -1; @@ -1493,55 +1397,41 @@ rspamd_register_symbol_fromlua (lua_State *L, } if (ref != -1) { - if (type & SYMBOL_TYPE_USE_CORO) { - /* Coroutines are incompatible with squeezing */ - no_squeeze = TRUE; + cd = rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct lua_callback_data)); + cd->magic = rspamd_lua_callback_magic; + cd->cb_is_ref = TRUE; + cd->callback.ref = ref; + cd->L = L; + + if (name) { + cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); } - /* - * We call for routine called lua_squeeze_rules.squeeze_rule if it exists - */ - if (no_squeeze || (ret = rspamd_lua_squeeze_rule (L, cfg, name, ref, - type, parent)) == -1) { - cd = rspamd_mempool_alloc0 (cfg->cfg_pool, - sizeof (struct lua_callback_data)); - cd->magic = rspamd_lua_callback_magic; - cd->cb_is_ref = TRUE; - cd->callback.ref = ref; - cd->L = L; - - if (name) { - cd->symbol = rspamd_mempool_strdup (cfg->cfg_pool, name); - } - if (type & SYMBOL_TYPE_USE_CORO) { - ret = rspamd_symcache_add_symbol (cfg->cache, - name, - priority, - lua_metric_symbol_callback_coro, - cd, - type, - parent); - } - else { - ret = rspamd_symcache_add_symbol (cfg->cache, - name, - priority, - lua_metric_symbol_callback, - cd, - type, - parent); - } - rspamd_mempool_add_destructor (cfg->cfg_pool, - (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, - cd); + if (type & SYMBOL_TYPE_USE_CORO) { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback_coro, + cd, + type, + parent); + } + else { + ret = rspamd_symcache_add_symbol (cfg->cache, + name, + priority, + lua_metric_symbol_callback, + cd, + type, + parent); } + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t)lua_destroy_cfg_symbol, + cd); } else { - if (!no_squeeze) { - rspamd_lua_squeeze_rule (L, cfg, name, ref, - type, parent); - } - /* Not a squeezed symbol */ + /* No callback */ ret = rspamd_symcache_add_symbol (cfg->cache, name, priority, @@ -1587,8 +1477,7 @@ lua_config_register_post_filter (lua_State *L) order, SYMBOL_TYPE_POSTFILTER|SYMBOL_TYPE_CALLBACK, -1, - FALSE, - TRUE); + FALSE); lua_pushboolean (L, ret); } @@ -1632,8 +1521,7 @@ lua_config_register_pre_filter (lua_State *L) order, SYMBOL_TYPE_PREFILTER|SYMBOL_TYPE_CALLBACK, -1, - FALSE, - TRUE); + FALSE); lua_pushboolean (L, ret); } @@ -1694,9 +1582,6 @@ lua_parse_symbol_flags (const gchar *str) if (strstr (str, "idempotent") != NULL) { ret |= SYMBOL_TYPE_IDEMPOTENT; } - if (strstr (str, "squeezed") != NULL) { - ret |= SYMBOL_TYPE_SQUEEZED; - } if (strstr (str, "trivial") != NULL) { ret |= SYMBOL_TYPE_TRIVIAL; } @@ -1787,11 +1672,6 @@ lua_push_symbol_flags (lua_State *L, guint flags) lua_rawseti (L, -2, i++); } - if (flags & SYMBOL_TYPE_SQUEEZED) { - lua_pushstring (L, "squeezed"); - lua_rawseti (L, -2, i++); - } - if (flags & SYMBOL_TYPE_EXPLICIT_DISABLE) { lua_pushstring (L, "explicit_disable"); lua_rawseti (L, -2, i++); @@ -1893,7 +1773,7 @@ lua_config_register_symbol (lua_State * L) const gchar *name = NULL, *flags_str = NULL, *type_str = NULL, *description = NULL, *group = NULL; double weight = 0, score = NAN, parent_float = NAN; - gboolean one_shot = FALSE, no_squeeze = FALSE; + gboolean one_shot = FALSE; gint ret = -1, cbref = -1, type, flags = 0; gint64 parent = 0, priority = 0, nshots = 0; GError *err = NULL; @@ -1901,20 +1781,16 @@ lua_config_register_symbol (lua_State * L) if (cfg) { if (!rspamd_lua_parse_table_arguments (L, 2, &err, "name=S;weight=N;callback=F;flags=S;type=S;priority=I;parent=D;" - "score=D;description=S;group=S;one_shot=B;nshots=I;no_squeeze=B", + "score=D;description=S;group=S;one_shot=B;nshots=I", &name, &weight, &cbref, &flags_str, &type_str, &priority, &parent_float, - &score, &description, &group, &one_shot, &nshots, &no_squeeze)) { + &score, &description, &group, &one_shot, &nshots)) { msg_err_config ("bad arguments: %e", err); g_error_free (err); return luaL_error (L, "invalid arguments"); } - if (!no_squeeze) { - no_squeeze = cfg->disable_lua_squeeze; - } - if (nshots == 0) { nshots = cfg->default_max_shots; } @@ -1929,9 +1805,6 @@ lua_config_register_symbol (lua_State * L) } if (flags_str) { - /* Turn off squeezing as well for now */ - /* TODO: deal with it */ - no_squeeze = TRUE; type |= lua_parse_symbol_flags (flags_str); } @@ -1950,8 +1823,7 @@ lua_config_register_symbol (lua_State * L) priority, type, parent, - FALSE, - no_squeeze); + FALSE); if (!isnan (score) || group) { if (one_shot) { @@ -2039,7 +1911,6 @@ lua_config_register_symbols (lua_State *L) 0, SYMBOL_TYPE_CALLBACK, -1, - FALSE, FALSE); for (i = top; i <= lua_gettop (L); i++) { @@ -2131,8 +2002,7 @@ lua_config_register_callback_symbol (lua_State * L) 0, SYMBOL_TYPE_CALLBACK, -1, - FALSE, - lua_type (L, top + 1) == LUA_TSTRING); + FALSE); } lua_pushinteger (L, ret); @@ -2174,8 +2044,7 @@ lua_config_register_callback_symbol_priority (lua_State * L) priority, SYMBOL_TYPE_CALLBACK, -1, - FALSE, - lua_type (L, top + 2) == LUA_TSTRING); + FALSE); } lua_pushinteger (L, ret); @@ -2183,40 +2052,6 @@ lua_config_register_callback_symbol_priority (lua_State * L) return 1; } -static gboolean -rspamd_lua_squeeze_dependency (lua_State *L, struct rspamd_config *cfg, - const gchar *child, - const gchar *parent) -{ - gint err_idx; - gboolean ret = FALSE; - - g_assert (parent != NULL); - g_assert (child != NULL); - - lua_pushcfunction (L, &rspamd_lua_traceback); - err_idx = lua_gettop (L); - - if (rspamd_lua_require_function (L, "lua_squeeze_rules", "squeeze_dependency")) { - lua_pushstring (L, child); - lua_pushstring (L, parent); - - if (lua_pcall (L, 2, 1, err_idx) != 0) { - msg_err_config ("call to squeeze_dependency script failed: %s", - lua_tostring (L, -1)); - } - else { - ret = lua_toboolean (L, -1); - } - } - else { - msg_err_config ("cannot get lua_squeeze_rules.squeeze_dependency function"); - } - - lua_settop (L, err_idx - 1); - - return ret; -} static gint lua_config_register_dependency (lua_State * L) @@ -2225,50 +2060,31 @@ lua_config_register_dependency (lua_State * L) struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *parent = NULL, *child = NULL; gint child_id; - gboolean skip_squeeze; if (cfg == NULL) { lua_error (L); return 0; } - skip_squeeze = cfg->disable_lua_squeeze; - if (lua_type (L, 2) == LUA_TNUMBER) { child_id = luaL_checknumber (L, 2); parent = luaL_checkstring (L, 3); - if (!skip_squeeze && lua_isboolean (L, 4)) { - skip_squeeze = lua_toboolean (L, 4); - } - msg_warn_config ("calling for obsolete method to register deps for symbol %d->%s", child_id, parent); if (child_id > 0 && parent != NULL) { - if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, - rspamd_symcache_symbol_by_id (cfg->cache, child_id), - parent)) { - rspamd_symcache_add_dependency (cfg->cache, child_id, parent); - } + rspamd_symcache_add_dependency (cfg->cache, child_id, parent); } } else { child = luaL_checkstring (L,2); parent = luaL_checkstring (L, 3); - if (!skip_squeeze && lua_isboolean (L, 4)) { - skip_squeeze = lua_toboolean (L, 4); - } - if (child != NULL && parent != NULL) { - - if (skip_squeeze || !rspamd_lua_squeeze_dependency (L, cfg, child, parent)) { - rspamd_symcache_add_delayed_dependency (cfg->cache, child, - parent); - } - + rspamd_symcache_add_delayed_dependency (cfg->cache, child, + parent); } } @@ -2609,12 +2425,11 @@ lua_config_newindex (lua_State *L) struct rspamd_config *cfg = lua_check_config (L, 1); const gchar *name; gint id, nshots, flags = 0; - gboolean optional = FALSE, no_squeeze = FALSE; + gboolean optional = FALSE; name = luaL_checkstring (L, 2); if (cfg != NULL && name != NULL && lua_gettop (L) == 3) { - no_squeeze = cfg->disable_lua_squeeze; if (lua_type (L, 3) == LUA_TFUNCTION) { /* Normal symbol from just a function */ @@ -2627,15 +2442,13 @@ lua_config_newindex (lua_State *L) 0, SYMBOL_TYPE_NORMAL, -1, - FALSE, - no_squeeze); + FALSE); } else if (lua_type (L, 3) == LUA_TTABLE) { gint type = SYMBOL_TYPE_NORMAL, priority = 0, idx; gdouble weight = 1.0, score = NAN; const char *type_str, *group = NULL, *description = NULL; - no_squeeze = cfg->disable_lua_squeeze; /* * Table can have the following attributes: * "callback" - should be a callback function @@ -2704,24 +2517,6 @@ lua_config_newindex (lua_State *L) } lua_pop (L, 1); - lua_pushstring (L, "condition"); - lua_gettable (L, -2); - - if (lua_type (L, -1) == LUA_TFUNCTION) { - no_squeeze = TRUE; - } - lua_pop (L, 1); - - if (!no_squeeze) { - lua_pushstring (L, "no_squeeze"); - lua_gettable (L, -2); - - if (lua_toboolean (L, -1)) { - no_squeeze = TRUE; - } - lua_pop (L, 1); - } - id = rspamd_register_symbol_fromlua (L, cfg, name, @@ -2730,8 +2525,7 @@ lua_config_newindex (lua_State *L) priority, type, -1, - optional, - no_squeeze); + optional); if (id != -1) { /* Check for condition */ diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index 5c5d63c55..48dca67b9 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -24,7 +24,6 @@ end local rspamd_logger = require "rspamd_logger" local rspamd_maps = require "lua_maps" -local lua_squeeze = require "lua_squeeze_rules" local lua_util = require "lua_util" local rspamd_util = require "rspamd_util" local rspamd_ip = require "rspamd_ip" @@ -46,7 +45,6 @@ local selectors_cache = {} -- Used to speed up selectors in settings local function apply_settings(task, to_apply) task:set_settings(to_apply) task:cache_set('settings', to_apply) - lua_squeeze.handle_settings(task, to_apply) if to_apply['add_headers'] or to_apply['remove_headers'] then local rep = { -- 2.39.5