Browse Source

[Fix] Neural: Distinguish missing symbols from symbols with low scores

tags/2.4
Vsevolod Stakhov 4 years ago
parent
commit
a8d97d481a
2 changed files with 10 additions and 3 deletions
  1. 9
    2
      src/lua/lua_task.c
  2. 1
    1
      src/plugins/lua/neural.lua

+ 9
- 2
src/lua/lua_task.c View File

LUA_FUNCTION_DEF (task, get_symbols_tokens); LUA_FUNCTION_DEF (task, get_symbols_tokens);


/*** /***
* @method task:process_ann_tokens(symbols, ann_tokens, offset)
* @method task:process_ann_tokens(symbols, ann_tokens, offset, [min])
* Processes ann tokens * Processes ann tokens
* @param {table|string} symbols list of symbols in this profile * @param {table|string} symbols list of symbols in this profile
* @param {table|number} ann_tokens list of tokens (including metatokens) * @param {table|number} ann_tokens list of tokens (including metatokens)
* @param {integer} offset offset for symbols token (#metatokens) * @param {integer} offset offset for symbols token (#metatokens)
* @param {number} min minimum value for symbols found (e.g. for 0 score symbols)
* @return nothing * @return nothing
*/ */
LUA_FUNCTION_DEF (task, process_ann_tokens); LUA_FUNCTION_DEF (task, process_ann_tokens);
LUA_TRACE_POINT; LUA_TRACE_POINT;
struct rspamd_task *task = lua_check_task (L, 1); struct rspamd_task *task = lua_check_task (L, 1);
gint offset = luaL_checkinteger (L, 4); gint offset = luaL_checkinteger (L, 4);
gdouble min_score = 0.0;


if (task && lua_istable (L, 2) && lua_istable (L, 3)) { if (task && lua_istable (L, 2) && lua_istable (L, 3)) {
guint symlen = rspamd_lua_table_size (L, 2); guint symlen = rspamd_lua_table_size (L, 2);
if (lua_isnumber (L, 5)) {
min_score = lua_tonumber (L, 5);
}


for (guint i = 1; i <= symlen; i ++, offset ++) { for (guint i = 1; i <= symlen; i ++, offset ++) {
const gchar *sym; const gchar *sym;
if (!isnan (sres->score) && !isinf (sres->score) && if (!isnan (sres->score) && !isinf (sres->score) &&
(!sres->sym || (!sres->sym ||
!(rspamd_symcache_item_flags (sres->sym->cache_item) & SYMBOL_TYPE_NOSTAT))) { !(rspamd_symcache_item_flags (sres->sym->cache_item) & SYMBOL_TYPE_NOSTAT))) {
gdouble norm_score = fabs (tanh (sres->score));



lua_pushnumber (L, fabs (tanh (sres->score)));
lua_pushnumber (L, MAX (min_score , norm_score));
lua_rawseti (L, 3, offset + 1); lua_rawseti (L, 3, offset + 1);
} }
} }

+ 1
- 1
src/plugins/lua/neural.lua View File

vec[i] = v vec[i] = v
end end


task:process_ann_tokens(profile.symbols, vec, #mt)
task:process_ann_tokens(profile.symbols, vec, #mt, 0.1)


return vec return vec
end end

Loading…
Cancel
Save