diff options
| -rw-r--r-- | conf/scores.d/surbl_group.conf | 4 | ||||
| -rw-r--r-- | lualib/lua_bayes_learn.lua | 13 | ||||
| -rw-r--r-- | rules/misc.lua | 12 | ||||
| -rw-r--r-- | src/libstat/stat_process.c | 20 | ||||
| -rw-r--r-- | src/lua/lua_config.c | 852 | ||||
| -rw-r--r-- | src/plugins/lua/history_redis.lua | 4 | ||||
| -rw-r--r-- | test/functional/lib/rspamd.robot | 17 |
7 files changed, 469 insertions, 453 deletions
diff --git a/conf/scores.d/surbl_group.conf b/conf/scores.d/surbl_group.conf index 89579ca15..255c03214 100644 --- a/conf/scores.d/surbl_group.conf +++ b/conf/scores.d/surbl_group.conf @@ -214,13 +214,13 @@ symbols = { groups = ["uribl"]; } "URIBL_RED" { - weight = 3.5; + weight = 0.5; description = "A domain in the message is listed in URIBL.com red"; one_shot = true; groups = ["uribl"]; } "URIBL_GREY" { - weight = 1.5; + weight = 2.5; description = "A domain in the message is listed in URIBL.com grey"; one_shot = true; groups = ["uribl"]; diff --git a/lualib/lua_bayes_learn.lua b/lualib/lua_bayes_learn.lua index ea97db6f8..82f044d7d 100644 --- a/lualib/lua_bayes_learn.lua +++ b/lualib/lua_bayes_learn.lua @@ -18,6 +18,7 @@ limitations under the License. local lua_util = require "lua_util" local lua_verdict = require "lua_verdict" +local logger = require "rspamd_logger" local N = "lua_bayes" local exports = {} @@ -56,16 +57,16 @@ exports.autolearn = function(task, conf) local mime_rcpts = 'undef' local mr = task:get_recipients('mime') if mr then + local r_addrs = {} for _, r in ipairs(mr) do - if mime_rcpts == 'undef' then - mime_rcpts = r.addr - else - mime_rcpts = mime_rcpts .. ',' .. r.addr - end + r_addrs[#r_addrs + 1] = r.addr + end + if #r_addrs > 0 then + mime_rcpts = table.concat(r_addrs, ',') end end - lua_util.debugm(N, task, 'id: %s, from: <%s>: can autolearn %s: score %s %s %s, mime_rcpts: <%s>', + logger.info(task, 'id: %s, from: <%s>: can autolearn %s: score %s %s %s, mime_rcpts: <%s>', task:get_header('Message-Id') or '<undef>', from and from[1].addr or 'undef', verdict, diff --git a/rules/misc.lua b/rules/misc.lua index 8785ab4ac..4ddb00dfb 100644 --- a/rules/misc.lua +++ b/rules/misc.lua @@ -863,6 +863,9 @@ rspamd_config.COMPLETELY_EMPTY = { score = 15 } +-- Preserve compatibility +local rdns_auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, 'once_received', + false, false) -- Check for the hostname if it was not set local rnds_check_id = rspamd_config:register_symbol { name = 'RDNS_CHECK', @@ -899,6 +902,15 @@ local rnds_check_id = rspamd_config:register_symbol { -- TODO: settings might need to use this symbol if they depend on hostname... priority = lua_util.symbols_priorities.top - 1, description = 'Check if hostname has been resolved by MTA', + condition = function(task) + local task_ip = task:get_ip() + if ((not rdns_auth_and_local_conf[1] and task:get_user()) or + (not rdns_auth_and_local_conf[2] and task_ip and task_ip:is_local())) then + return false + end + + return true + end } rspamd_config:register_symbol { diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index ad976e713..5db3af6ce 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -509,6 +509,14 @@ rspamd_stat_classify(struct rspamd_task *task, lua_State *L, unsigned int stage, return ret; } + if (task->message == NULL) { + ret = RSPAMD_STAT_PROCESS_ERROR; + msg_err_task("trying to classify empty message"); + + task->processed_stages |= stage; + return ret; + } + if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS_PRE) { /* Preprocess tokens */ rspamd_stat_preprocess(st_ctx, task, FALSE, FALSE); @@ -892,6 +900,18 @@ rspamd_stat_learn(struct rspamd_task *task, return ret; } + + if (task->message == NULL) { + ret = RSPAMD_STAT_PROCESS_ERROR; + if (err && *err == NULL) { + g_set_error(err, rspamd_stat_quark(), 500, + "Trying to learn an empty message"); + } + + task->processed_stages |= stage; + return ret; + } + if (stage == RSPAMD_TASK_STAGE_LEARN_PRE) { /* Process classifiers */ rspamd_stat_preprocess(st_ctx, task, TRUE, spam); diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 717aa81ce..be4dd7081 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -34,7 +34,11 @@ local function foo(task) -- do something end -rspamd_config:register_symbol('SYMBOL', 1.0, foo) +rspamd_config:register_symbol{ + name = 'SYMBOL', + score = 1.0, + callback = foo +} -- Get configuration local tab = rspamd_config:get_all_opt('module') -- get table for module's options @@ -1957,210 +1961,454 @@ lua_config_get_symbol_flags(lua_State *L) return 1; } -static int -lua_config_register_symbol(lua_State *L) +static bool +lua_config_register_symbol_from_table(lua_State *L, struct rspamd_config *cfg, + const char *name, int tbl_idx, int *id_out) { - LUA_TRACE_POINT; - struct rspamd_config *cfg = lua_check_config(L, 1); - const char *name = NULL, *type_str = NULL, - *description = NULL, *group = NULL; - double weight = 0, score = NAN, parent_float = NAN; - gboolean one_shot = FALSE; - int ret = -1, cbref = -1; - unsigned int type = 0, flags = 0; - int64_t parent = 0, priority = 0, nshots = 0; + unsigned int type = SYMBOL_TYPE_NORMAL, priority = 0; + double weight = 1.0, score = NAN; + const char *type_str, *group = NULL, *description = NULL; GArray *allowed_ids = NULL, *forbidden_ids = NULL; - GError *err = NULL; - int prev_top = lua_gettop(L); + int id, nshots, cb_ref, parent = -1; + unsigned int flags = 0; + gboolean optional = FALSE; - if (cfg) { - if (!rspamd_lua_parse_table_arguments(L, 2, &err, - RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT, - "name=S;weight=N;callback=F;type=S;priority=I;parent=D;" - "score=D;description=S;group=S;one_shot=B;nshots=I", - &name, &weight, &cbref, &type_str, - &priority, &parent_float, - &score, &description, &group, &one_shot, &nshots)) { - msg_err_config("bad arguments: %e", err); - g_error_free(err); - lua_settop(L, prev_top); + /* + * Table can have the following attributes: + * "callback" - should be a callback function + * "weight" - optional weight + * "priority" - optional priority + * "type" - optional type (normal, virtual, callback) + * "flags" - optional flags + * -- Metric options + * "score" - optional default score (overridden by metric) + * "group" - optional default group + * "one_shot" - optional one shot mode + * "description" - optional description + */ + lua_pushvalue(L, tbl_idx); /* Push table on top of the stack */ - return luaL_error(L, "invalid arguments"); - } + if (name == NULL) { + /* Try to resolve name */ + lua_pushstring(L, "name"); + lua_gettable(L, -2); - /* Deal with flags and ids */ - lua_pushstring(L, "flags"); - lua_gettable(L, 2); - if (lua_type(L, -1) == LUA_TSTRING) { - flags = lua_parse_symbol_flags(lua_tostring(L, -1)); - } - else if (lua_type(L, -1) == LUA_TTABLE) { - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - flags |= lua_parse_symbol_flags(lua_tostring(L, -1)); - } - } - lua_pop(L, 1); /* Clean flags */ + if (lua_type(L, -1) != LUA_TSTRING) { + lua_pop(L, 2); + luaL_error(L, "name is not specified"); - lua_pushstring(L, "allowed_ids"); - lua_gettable(L, 2); - if (lua_type(L, -1) == LUA_TSTRING) { - allowed_ids = rspamd_process_id_list(lua_tostring(L, -1)); + return false; } - else if (lua_type(L, -1) == LUA_TTABLE) { - allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), - rspamd_lua_table_size(L, -1)); - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - uint32_t v = lua_tointeger(L, -1); - g_array_append_val(allowed_ids, v); - } + else { + name = lua_tostring(L, -1); } + lua_pop(L, 1); + } - lua_pushstring(L, "forbidden_ids"); - lua_gettable(L, 2); - if (lua_type(L, -1) == LUA_TSTRING) { - forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1)); + lua_pushstring(L, "callback"); + lua_gettable(L, -2); + + if (lua_type(L, -1) != LUA_TFUNCTION) { + cb_ref = -1; + } + else { + cb_ref = luaL_ref(L, LUA_REGISTRYINDEX); + } + lua_pop(L, 1); + + /* Optional fields */ + lua_pushstring(L, "weight"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TNUMBER) { + weight = lua_tonumber(L, -1); + } + lua_pop(L, 1); + + lua_pushstring(L, "priority"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TNUMBER) { + priority = lua_tointeger(L, -1); + } + lua_pop(L, 1); + + lua_pushstring(L, "optional"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TBOOLEAN) { + optional = lua_toboolean(L, -1); + } + lua_pop(L, 1); + + lua_pushstring(L, "type"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TSTRING) { + type_str = lua_tostring(L, -1); + } + else { + type_str = "normal"; + } + lua_pop(L, 1); + + type = lua_parse_symbol_type(type_str); + + if (!name && !(type & SYMBOL_TYPE_CALLBACK)) { + luaL_error(L, "no symbol name but type is not callback"); + + return false; + } + else if (!(type & SYMBOL_TYPE_VIRTUAL) && cb_ref == -1) { + luaL_error(L, "no callback for symbol %s", name); + + return false; + } + + lua_pushstring(L, "parent"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TNUMBER) { + parent = lua_tointeger(L, -1); + } + lua_pop(L, 1); + + + /* Deal with flags and ids */ + lua_pushstring(L, "flags"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + flags = lua_parse_symbol_flags(lua_tostring(L, -1)); + } + else if (lua_type(L, -1) == LUA_TTABLE) { + for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { + flags |= lua_parse_symbol_flags(lua_tostring(L, -1)); } - else if (lua_type(L, -1) == LUA_TTABLE) { - forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), - rspamd_lua_table_size(L, -1)); - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - uint32_t v = lua_tointeger(L, -1); - g_array_append_val(forbidden_ids, v); - } + } + lua_pop(L, 1); /* Clean flags */ + + lua_pushstring(L, "allowed_ids"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + allowed_ids = rspamd_process_id_list(lua_tostring(L, -1)); + } + else if (lua_type(L, -1) == LUA_TTABLE) { + allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), + rspamd_lua_table_size(L, -1)); + for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { + uint32_t v = lua_tointeger(L, -1); + g_array_append_val(allowed_ids, v); } - lua_pop(L, 1); + } + lua_pop(L, 1); - if (nshots == 0) { - nshots = cfg->default_max_shots; + lua_pushstring(L, "forbidden_ids"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1)); + } + else if (lua_type(L, -1) == LUA_TTABLE) { + forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), + rspamd_lua_table_size(L, -1)); + for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { + uint32_t v = lua_tointeger(L, -1); + g_array_append_val(forbidden_ids, v); } + } + lua_pop(L, 1); - type = lua_parse_symbol_type(type_str); + id = rspamd_register_symbol_fromlua(L, + cfg, + name, + cb_ref, + weight, + priority, + type | flags, + parent, + allowed_ids, forbidden_ids, + optional); - if (!name && !(type & SYMBOL_TYPE_CALLBACK)) { - lua_settop(L, prev_top); - return luaL_error(L, "no symbol name but type is not callback"); - } - else if (!(type & SYMBOL_TYPE_VIRTUAL) && cbref == -1) { - lua_settop(L, prev_top); - return luaL_error(L, "no callback for symbol %s", name); - } + if (allowed_ids) { + g_array_free(allowed_ids, TRUE); + } + + if (forbidden_ids) { + g_array_free(forbidden_ids, TRUE); + } + + if (id != -1) { + if (cb_ref != -1) { + /* Check for condition */ + lua_pushstring(L, "condition"); + lua_gettable(L, -2); - if (isnan(parent_float)) { - parent = -1; + if (lua_type(L, -1) == LUA_TFUNCTION) { + int condref; + + /* Here we pop function from the stack, so no lua_pop is required */ + condref = luaL_ref(L, LUA_REGISTRYINDEX); + g_assert(name != NULL); + rspamd_symcache_add_condition_delayed(cfg->cache, + name, L, condref); + } + else { + lua_pop(L, 1); + } } - else { - parent = parent_float; + + /* Check for augmentations */ + lua_pushstring(L, "augmentations"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TTABLE) { + + int aug_tbl_idx = lua_gettop(L); + for (lua_pushnil(L); lua_next(L, aug_tbl_idx); lua_pop(L, 1)) { + rspamd_symcache_add_symbol_augmentation(cfg->cache, id, + lua_tostring(L, -1), NULL); + } } - ret = rspamd_register_symbol_fromlua(L, - cfg, - name, - cbref, - weight == 0 ? 1.0 : weight, - priority, - type | flags, - parent, - allowed_ids, forbidden_ids, - FALSE); + lua_pop(L, 1); + } - if (allowed_ids) { - g_array_free(allowed_ids, TRUE); + /* + * Now check if a symbol has not been registered in any metric and + * insert default value if applicable + */ + struct rspamd_symbol *sym = g_hash_table_lookup(cfg->symbols, name); + if (sym == NULL || (sym->flags & RSPAMD_SYMBOL_FLAG_UNSCORED)) { + nshots = cfg->default_max_shots; + + lua_pushstring(L, "score"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TNUMBER) { + score = lua_tonumber(L, -1); + + if (sym) { + /* Reset unscored flag */ + sym->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED; + } } + lua_pop(L, 1); - if (forbidden_ids) { - g_array_free(forbidden_ids, TRUE); + lua_pushstring(L, "group"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + group = lua_tostring(L, -1); } + lua_pop(L, 1); + + if (!isnan(score) || group != NULL) { + lua_pushstring(L, "description"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TSTRING) { + description = lua_tostring(L, -1); + } + lua_pop(L, 1); + + lua_pushstring(L, "one_shot"); + lua_gettable(L, -2); - if (ret != -1) { - if (!isnan(score) || group) { - if (one_shot) { + if (lua_type(L, -1) == LUA_TBOOLEAN) { + if (lua_toboolean(L, -1)) { nshots = 1; } + } + lua_pop(L, 1); - rspamd_config_add_symbol(cfg, name, - score, description, group, flags, - 0, nshots); + lua_pushstring(L, "one_param"); + lua_gettable(L, -2); - lua_pushstring(L, "groups"); - lua_gettable(L, 2); + if (lua_type(L, -1) == LUA_TBOOLEAN) { + if (lua_toboolean(L, -1)) { + flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; + } + } + lua_pop(L, 1); - if (lua_istable(L, -1)) { - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - if (lua_isstring(L, -1)) { - rspamd_config_add_symbol_group(cfg, name, - lua_tostring(L, -1)); - } - else { - lua_settop(L, prev_top); - return luaL_error(L, "invalid groups element"); - } + /* + * Do not override the existing symbols (using zero priority), + * since we are defining default values here + */ + if (!isnan(score)) { + rspamd_config_add_symbol(cfg, name, score, + description, group, flags, 0, nshots); + } + else if (group) { + /* Add with zero score */ + rspamd_config_add_symbol(cfg, name, NAN, + description, group, flags, 0, nshots); + } + + lua_pushstring(L, "groups"); + lua_gettable(L, -2); + + if (lua_istable(L, -1)) { + for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { + if (lua_isstring(L, -1)) { + rspamd_config_add_symbol_group(cfg, name, + lua_tostring(L, -1)); + } + else { + lua_pop(L, 2); + luaL_error(L, "invalid groups element"); + + return false; } } + } - lua_pop(L, 1); + lua_pop(L, 1); + } + } + else { + /* Fill in missing fields from lua definition if they are not set */ + if (sym->description == NULL) { + lua_pushstring(L, "description"); + lua_gettable(L, -2); + + if (lua_type(L, -1) == LUA_TSTRING) { + description = lua_tostring(L, -1); } + lua_pop(L, 1); - lua_pushstring(L, "augmentations"); - lua_gettable(L, 2); + if (description) { + sym->description = rspamd_mempool_strdup(cfg->cfg_pool, description); + } + } - if (lua_type(L, -1) == LUA_TTABLE) { - int tbl_idx = lua_gettop(L); - for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) { - size_t len; - const char *augmentation = lua_tolstring(L, -1, &len), *eqsign_pos; + /* If ungrouped and there is a group defined in lua, change the primary group + * Otherwise, add to the list of groups for this symbol. */ + lua_pushstring(L, "group"); + lua_gettable(L, -2); + if (lua_type(L, -1) == LUA_TSTRING) { + group = lua_tostring(L, -1); + } + lua_pop(L, 1); + if (group) { + if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) { + /* Unset the "ungrouped" group */ + sym->gr = NULL; + } + /* Add the group. If the symbol was ungrouped, this will + * clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */ + rspamd_config_add_symbol_group(cfg, name, group); + } + } - /* Find `=` symbol and use it as a separator */ - eqsign_pos = memchr(augmentation, '=', len); - if (eqsign_pos != NULL && eqsign_pos + 1 < augmentation + len) { - rspamd_ftok_t tok; + /* Remove table from stack */ + lua_pop(L, 1); - tok.begin = augmentation; - tok.len = eqsign_pos - augmentation; - char *augentation_name = rspamd_ftokdup(&tok); + *id_out = id; - tok.begin = eqsign_pos + 1; - tok.len = (augmentation + len) - tok.begin; + return true; +} - char *augmentation_value = rspamd_ftokdup(&tok); +/* Legacy symbol registration */ +static bool +lua_config_register_symbol_legacy(lua_State *L, struct rspamd_config *cfg, int pos, int *id_out) +{ + const char *name = NULL, *type_str = NULL, + *description = NULL, *group = NULL; + double weight = 0, score = NAN, parent_float = NAN; + gboolean one_shot = FALSE; + int ret, cbref = -1; + unsigned int type = 0, flags = 0; + int64_t parent = 0, priority = 0, nshots = 0; - if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret, - augentation_name, augmentation_value)) { - lua_settop(L, prev_top); - g_free(augmentation_value); - g_free(augentation_name); + GError *err = NULL; + if (!rspamd_lua_parse_table_arguments(L, pos, &err, + RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT, + "name=S;weight=N;callback=F;type=S;priority=I;parent=D;" + "score=D;description=S;group=S;one_shot=B;nshots=I", + &name, &weight, &cbref, &type_str, + &priority, &parent_float, + &score, &description, &group, &one_shot, &nshots)) { + msg_err_config("bad arguments: %e", err); + g_error_free(err); - return luaL_error(L, "unknown or invalid augmentation %s in symbol %s", - augmentation, name); - } + return false; + } - g_free(augmentation_value); - g_free(augentation_name); - } - else { - /* Just a value */ - if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret, - augmentation, NULL)) { - lua_settop(L, prev_top); + type = lua_parse_symbol_type(type_str); - return luaL_error(L, "unknown augmentation %s in symbol %s", - augmentation, name); - } - } - } + if (!name && !(type & SYMBOL_TYPE_CALLBACK)) { + luaL_error(L, "no symbol name but type is not callback"); + + return false; + } + else if (!(type & SYMBOL_TYPE_VIRTUAL) && cbref == -1) { + luaL_error(L, "no callback for symbol %s", name); + + return false; + } + + if (isnan(parent_float)) { + parent = -1; + } + else { + parent = parent_float; + } + + ret = rspamd_register_symbol_fromlua(L, + cfg, + name, + cbref, + weight == 0 ? 1.0 : weight, + priority, + type | flags, + parent, + NULL, NULL, + FALSE); + + if (ret != -1) { + if (!isnan(score) || group) { + if (one_shot) { + nshots = 1; + } + if (nshots == 0) { + nshots = cfg->default_max_shots; } + + rspamd_config_add_symbol(cfg, name, score, + description, group, flags, 0, nshots); } + + *id_out = ret; + + return true; } - else { - lua_settop(L, prev_top); - return luaL_error(L, "invalid arguments"); + return false; +} + +static int +lua_config_register_symbol(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + int id = -1; + + if (lua_type(L, 2) == LUA_TSTRING) { + if (lua_config_register_symbol_legacy(L, cfg, 2, &id)) { + lua_pushinteger(L, id); + + return 1; + } + else { + return luaL_error(L, "bad arguments"); + } } + else if (lua_config_register_symbol_from_table(L, cfg, NULL, 2, &id)) { + lua_pushinteger(L, id); - lua_settop(L, prev_top); - lua_pushinteger(L, ret); + return 1; + } - return 1; + return 0; } static int @@ -2647,10 +2895,7 @@ lua_config_newindex(lua_State *L) LUA_TRACE_POINT; struct rspamd_config *cfg = lua_check_config(L, 1); const char *name; - GArray *allowed_ids = NULL, *forbidden_ids = NULL; - int id, nshots; - unsigned int flags = 0; - gboolean optional = FALSE; + int id = -1; name = luaL_checkstring(L, 2); @@ -2671,291 +2916,14 @@ lua_config_newindex(lua_State *L) FALSE); } else if (lua_type(L, 3) == LUA_TTABLE) { - unsigned int type = SYMBOL_TYPE_NORMAL, priority = 0; - int idx; - double weight = 1.0, score = NAN; - const char *type_str, *group = NULL, *description = NULL; - - /* - * Table can have the following attributes: - * "callback" - should be a callback function - * "weight" - optional weight - * "priority" - optional priority - * "type" - optional type (normal, virtual, callback) - * "flags" - optional flags - * -- Metric options - * "score" - optional default score (overridden by metric) - * "group" - optional default group - * "one_shot" - optional one shot mode - * "description" - optional description - */ - lua_pushvalue(L, 3); - lua_pushstring(L, "callback"); - lua_gettable(L, -2); - - if (lua_type(L, -1) != LUA_TFUNCTION) { - lua_pop(L, 2); - msg_info_config("cannot find callback definition for %s", - name); - return 0; - } - idx = luaL_ref(L, LUA_REGISTRYINDEX); - - /* Optional fields */ - lua_pushstring(L, "weight"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TNUMBER) { - weight = lua_tonumber(L, -1); - } - lua_pop(L, 1); - - lua_pushstring(L, "priority"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TNUMBER) { - priority = lua_tointeger(L, -1); - } - lua_pop(L, 1); - - lua_pushstring(L, "optional"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TBOOLEAN) { - optional = lua_toboolean(L, -1); - } - lua_pop(L, 1); - - lua_pushstring(L, "type"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TSTRING) { - type_str = lua_tostring(L, -1); - type = lua_parse_symbol_type(type_str); - } - lua_pop(L, 1); - - /* Deal with flags and ids */ - lua_pushstring(L, "flags"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) { - flags = lua_parse_symbol_flags(lua_tostring(L, -1)); - } - else if (lua_type(L, -1) == LUA_TTABLE) { - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - flags |= lua_parse_symbol_flags(lua_tostring(L, -1)); - } - } - lua_pop(L, 1); /* Clean flags */ - - lua_pushstring(L, "allowed_ids"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) { - allowed_ids = rspamd_process_id_list(lua_tostring(L, -1)); - } - else if (lua_type(L, -1) == LUA_TTABLE) { - allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), - rspamd_lua_table_size(L, -1)); - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - uint32_t v = lua_tointeger(L, -1); - g_array_append_val(allowed_ids, v); - } - } - lua_pop(L, 1); - - lua_pushstring(L, "forbidden_ids"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) { - forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1)); - } - else if (lua_type(L, -1) == LUA_TTABLE) { - forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(uint32_t), - rspamd_lua_table_size(L, -1)); - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - uint32_t v = lua_tointeger(L, -1); - g_array_append_val(forbidden_ids, v); - } - } - lua_pop(L, 1); - - id = rspamd_register_symbol_fromlua(L, - cfg, - name, - idx, - weight, - priority, - type | flags, - -1, - allowed_ids, forbidden_ids, - optional); - - if (allowed_ids) { - g_array_free(allowed_ids, TRUE); - } - - if (forbidden_ids) { - g_array_free(forbidden_ids, TRUE); - } - - if (id != -1) { - /* Check for condition */ - lua_pushstring(L, "condition"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TFUNCTION) { - int condref; - - /* Here we pop function from the stack, so no lua_pop is required */ - condref = luaL_ref(L, LUA_REGISTRYINDEX); - g_assert(name != NULL); - rspamd_symcache_add_condition_delayed(cfg->cache, - name, L, condref); - } - else { - lua_pop(L, 1); - } - - /* Check for augmentations */ - lua_pushstring(L, "augmentations"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TTABLE) { - - int tbl_idx = lua_gettop(L); - for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) { - rspamd_symcache_add_symbol_augmentation(cfg->cache, id, - lua_tostring(L, -1), NULL); - } - } - - lua_pop(L, 1); - } - - /* - * Now check if a symbol has not been registered in any metric and - * insert default value if applicable - */ - struct rspamd_symbol *sym = g_hash_table_lookup(cfg->symbols, name); - if (sym == NULL || (sym->flags & RSPAMD_SYMBOL_FLAG_UNSCORED)) { - nshots = cfg->default_max_shots; - - lua_pushstring(L, "score"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TNUMBER) { - score = lua_tonumber(L, -1); - - if (sym) { - /* Reset unscored flag */ - sym->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED; - } - } - lua_pop(L, 1); - - lua_pushstring(L, "group"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) { - group = lua_tostring(L, -1); - } - lua_pop(L, 1); - - if (!isnan(score) || group != NULL) { - lua_pushstring(L, "description"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TSTRING) { - description = lua_tostring(L, -1); - } - lua_pop(L, 1); - - lua_pushstring(L, "one_shot"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TBOOLEAN) { - if (lua_toboolean(L, -1)) { - nshots = 1; - } - } - lua_pop(L, 1); - - lua_pushstring(L, "one_param"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TBOOLEAN) { - if (lua_toboolean(L, -1)) { - flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM; - } - } - lua_pop(L, 1); - - /* - * Do not override the existing symbols (using zero priority), - * since we are defining default values here - */ - if (!isnan(score)) { - rspamd_config_add_symbol(cfg, name, score, - description, group, flags, 0, nshots); - } - else if (group) { - /* Add with zero score */ - rspamd_config_add_symbol(cfg, name, NAN, - description, group, flags, 0, nshots); - } - - lua_pushstring(L, "groups"); - lua_gettable(L, -2); - - if (lua_istable(L, -1)) { - for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) { - if (lua_isstring(L, -1)) { - rspamd_config_add_symbol_group(cfg, name, - lua_tostring(L, -1)); - } - else { - return luaL_error(L, "invalid groups element"); - } - } - } - - lua_pop(L, 1); - } - } - else { - /* Fill in missing fields from lua definition if they are not set */ - if (sym->description == NULL) { - lua_pushstring(L, "description"); - lua_gettable(L, -2); - - if (lua_type(L, -1) == LUA_TSTRING) { - description = lua_tostring(L, -1); - } - lua_pop(L, 1); - - if (description) { - sym->description = rspamd_mempool_strdup(cfg->cfg_pool, description); - } - } - - /* If ungrouped and there is a group defined in lua, change the primary group - * Otherwise, add to the list of groups for this symbol. */ - lua_pushstring(L, "group"); - lua_gettable(L, -2); - if (lua_type(L, -1) == LUA_TSTRING) { - group = lua_tostring(L, -1); - } - lua_pop(L, 1); - if (group) { - if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) { - /* Unset the "ungrouped" group */ - sym->gr = NULL; - } - /* Add the group. If the symbol was ungrouped, this will - * clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */ - rspamd_config_add_symbol_group(cfg, name, group); - } + /* Table symbol */ + if (lua_config_register_symbol_from_table(L, cfg, name, 3, &id)) { + lua_pushinteger(L, id); + return 1; } - - /* Remove table from stack */ - lua_pop(L, 1); + } + else { + return luaL_error(L, "invalid value for symbol"); } } else { diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index 3365b30cd..fff9f46b3 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -21,8 +21,8 @@ if confighelp then redis_history { # History key name key_prefix = 'rs_history{{HOSTNAME}}{{COMPRESS}}'; - # History expire in seconds - expire = 0; + # Expire in seconds for inactive keys, default to 5 days + expire = 432000; # History rows limit nrows = 200; # Use zstd compression when storing data in redis diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot index c45a19908..de4e5285f 100644 --- a/test/functional/lib/rspamd.robot +++ b/test/functional/lib/rspamd.robot @@ -345,7 +345,22 @@ Run Rspamd Export Scoped Variables ${RSPAMD_SCOPE} RSPAMD_PROCESS=${result} # Confirm worker is reachable - Wait Until Keyword Succeeds 15x 1 sec Ping Rspamd ${RSPAMD_LOCAL_ADDR} ${check_port} + FOR ${index} IN RANGE 37 + ${ok} = Rspamd Startup Check ${check_port} + IF ${ok} CONTINUE + Sleep 0.4s + END + +Rspamd Startup Check + [Arguments] ${check_port}=${RSPAMD_PORT_NORMAL} + ${handle} = Get Process Object + ${res} = Evaluate $handle.poll() + IF ${res} != None + ${stderr} = Get File ${RSPAMD_TMPDIR}/rspamd.stderr + Fail Process Is Gone, stderr: ${stderr} + END + ${ping} = Run Keyword And Return Status Ping Rspamd ${RSPAMD_LOCAL_ADDR} ${check_port} + [Return] ${ping} Rspamadm Setup ${RSPAMADM_TMPDIR} = Make Temporary Directory |