diff options
author | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2010-06-16 20:43:26 +0400 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2010-06-16 20:43:26 +0400 |
commit | a639bf512e3df778fa33c49d83c3996c9fe60d77 (patch) | |
tree | 072c4167dae82382b1c8eef93e0270339635e389 | |
parent | c4aab3053d2839e6d3b99f8a542b0a4f54f2b856 (diff) | |
download | rspamd-a639bf512e3df778fa33c49d83c3996c9fe60d77.tar.gz rspamd-a639bf512e3df778fa33c49d83c3996c9fe60d77.zip |
* Change metric logic
* Completely remove lex/yacc readers for config
* Make common sense of metric/action and symbols
* Sync changes with all plugins
TODO: add this to documentation
-rw-r--r-- | CMakeLists.txt | 49 | ||||
-rw-r--r-- | rspamd.xml.sample | 144 | ||||
-rw-r--r-- | src/cfg_file.h | 24 | ||||
-rw-r--r-- | src/cfg_utils.c | 71 | ||||
-rw-r--r-- | src/cfg_xml.c | 140 | ||||
-rw-r--r-- | src/cfg_xml.h | 3 | ||||
-rw-r--r-- | src/classifiers/winnow.c | 2 | ||||
-rw-r--r-- | src/filter.c | 234 | ||||
-rw-r--r-- | src/filter.h | 15 | ||||
-rw-r--r-- | src/lua/lua_common.c | 1 | ||||
-rw-r--r-- | src/lua/lua_config.c | 60 | ||||
-rw-r--r-- | src/lua/lua_task.c | 11 | ||||
-rw-r--r-- | src/main.c | 95 | ||||
-rw-r--r-- | src/plugins/chartable.c | 33 | ||||
-rw-r--r-- | src/plugins/emails.c | 26 | ||||
-rw-r--r-- | src/plugins/fuzzy_check.c | 38 | ||||
-rw-r--r-- | src/plugins/lua/forged_recipients.lua | 13 | ||||
-rw-r--r-- | src/plugins/lua/maillist.lua | 14 | ||||
-rw-r--r-- | src/plugins/lua/once_received.lua | 12 | ||||
-rw-r--r-- | src/plugins/lua/received_rbl.lua | 14 | ||||
-rw-r--r-- | src/plugins/lua/whitelist.lua | 13 | ||||
-rw-r--r-- | src/plugins/regexp.c | 41 | ||||
-rw-r--r-- | src/plugins/spf.c | 31 | ||||
-rw-r--r-- | src/plugins/surbl.c | 38 |
24 files changed, 335 insertions, 787 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 72bf8b97a..3ba509251 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,14 +78,6 @@ ELSE(NOT LUA_FOUND) INCLUDE_DIRECTORIES("${LUA_INCLUDE_DIR}") ENDIF(NOT LUA_FOUND) -# Lex and yacc -FIND_PROGRAM(LEX_EXECUTABLE lex) -FIND_PROGRAM(YACC_EXECUTABLE yacc) - -IF(NOT LEX_EXECUTABLE OR NOT YACC_EXECUTABLE) - MESSAGE(FATAL_ERROR "Error: yacc and lex are required for build") -ENDIF(NOT LEX_EXECUTABLE OR NOT YACC_EXECUTABLE) - IF(ENABLE_STATIC MATCHES "ON") pkg_check_modules(GLIB2 REQUIRED glib-2.0>=2.16) pkg_check_modules(PCRE REQUIRED libpcre) @@ -497,20 +489,8 @@ SET(UTILSDEPENDS src/mem_pool.c LIST(LENGTH PLUGINSSRC RSPAMD_MODULES_NUM) -SET(YACC_SRC src/cfg_file.y) -SET(LEX_SRC src/cfg_file.l) - -STRING(REPLACE "_file.y" "_yacc.c" YACC_OUTPUT ${YACC_SRC}) -STRING(REPLACE "_file.l" "_lex.c" LEX_OUTPUT ${LEX_SRC}) - ############################ TARGETS SECTION ############################### -ADD_CUSTOM_COMMAND(OUTPUT ${LEX_OUTPUT} - COMMAND ${LEX_EXECUTABLE} -o${LEX_OUTPUT} ${LEX_SRC} - DEPENDS ${LEX_SRC}) -ADD_CUSTOM_COMMAND(OUTPUT ${YACC_OUTPUT} - COMMAND ${YACC_EXECUTABLE} -d -o ${YACC_OUTPUT} ${YACC_SRC} - DEPENDS ${YACC_SRC}) ADD_CUSTOM_COMMAND(OUTPUT src/modules.c COMMAND ../utils/gen-modules.sh ${PLUGINSSRC} WORKING_DIRECTORY src) @@ -535,8 +515,7 @@ CONFIGURE_FILE(rspamd.xml.sample conf/rspamd.xml.sample @ONLY) ######################### LINK SECTION ############################### ADD_EXECUTABLE(rspamd ${RSPAMDSRC} ${CONTRIBSRC} ${TOKENIZERSSRC} - ${CLASSIFIERSSRC} ${PLUGINSSRC} ${YACC_OUTPUT} - ${LEX_OUTPUT}) + ${CLASSIFIERSSRC} ${PLUGINSSRC}) SET_TARGET_PROPERTIES(rspamd PROPERTIES LINKER_LANGUAGE C) SET_TARGET_PROPERTIES(rspamd PROPERTIES COMPILE_FLAGS "-DRSPAMD_MAIN") SET_TARGET_PROPERTIES(rspamd PROPERTIES VERSION ${RSPAMD_VERSION}) @@ -548,7 +527,6 @@ IF(ENABLE_PERL MATCHES "ON") IF(PERL_DYNALOADER) TARGET_LINK_LIBRARIES(rspamd dynaloader) ENDIF(PERL_DYNALOADER) - ADD_DEPENDENCIES(rspamd perlxs ${LEX_OUTPUT} ${YACC_OUTPUT}) ENDIF(ENABLE_PERL MATCHES "ON") IF(PERL_EXECUTABLE) @@ -669,31 +647,6 @@ IF(NOT EXISTS ${ETC_PREFIX}/rspamd/2tld.inc) INSTALL(FILES conf/2tld.inc DESTINATION ${ETC_PREFIX}/rspamd) ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/2tld.inc) -INSTALL(FILES conf/drugs.inc RENAME drugs.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) -IF(NOT EXISTS ${ETC_PREFIX}/rspamd/drugs.inc) -INSTALL(FILES conf/drugs.inc DESTINATION ${ETC_PREFIX}/rspamd) -ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/drugs.inc) - -INSTALL(FILES conf/fraud.inc RENAME fraud.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) -IF(NOT EXISTS ${ETC_PREFIX}/rspamd/fraud.inc) -INSTALL(FILES conf/fraud.inc DESTINATION ${ETC_PREFIX}/rspamd) -ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/fraud.inc) - -INSTALL(FILES conf/html.inc RENAME html.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) -IF(NOT EXISTS ${ETC_PREFIX}/rspamd/html.inc) -INSTALL(FILES conf/html.inc DESTINATION ${ETC_PREFIX}/rspamd) -ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/html.inc) - -INSTALL(FILES conf/headers.inc RENAME headers.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) -IF(NOT EXISTS ${ETC_PREFIX}/rspamd/headers.inc) -INSTALL(FILES conf/headers.inc DESTINATION ${ETC_PREFIX}/rspamd) -ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/headers.inc) - -INSTALL(FILES conf/lotto.inc RENAME lotto.inc.orig DESTINATION ${ETC_PREFIX}/rspamd) -IF(NOT EXISTS ${ETC_PREFIX}/rspamd/lotto.inc) -INSTALL(FILES conf/lotto.inc DESTINATION ${ETC_PREFIX}/rspamd) -ENDIF(NOT EXISTS ${ETC_PREFIX}/rspamd/lotto.inc) - INSTALL(FILES conf/rspamd.xml.sample DESTINATION ${ETC_PREFIX}/) # Lua plugins diff --git a/rspamd.xml.sample b/rspamd.xml.sample index 281ffced9..00d545908 100644 --- a/rspamd.xml.sample +++ b/rspamd.xml.sample @@ -19,77 +19,79 @@ <!-- Factors section --> -<factors> - <factor name="R_SPAM_FROM_MTU">8.00</factor> - <factor name="R_WWW_EKONF_COM">10.00</factor> - <factor name="R_TINYURL">2.00</factor> - <factor name="MISSING_SUBJECT">2.00</factor> - <factor name="FORGED_OUTLOOK_TAGS">2.10</factor> - <factor name="R_FAKE_THEBAT">8.00</factor> - <factor name="FORGED_SENDER">5.00</factor> - <factor name="DRUGS_MANYKINDS">2.00</factor> - <factor name="ADVANCE_FEE_2">3.30</factor> - <factor name="ADVANCE_FEE_3">2.12</factor> - <factor name="SUSPICIOUS_RECIPS">3.50</factor> - <factor name="FAKE_REPLY_C">6.00</factor> - <factor name="MIME_HTML_ONLY">1.00</factor> - <factor name="AB_SURBL_MULTI">5.50</factor> - <factor name="FORGED_MSGID_YAHOO">2.00</factor> - <factor name="SC_SURBL_MULTI">5.50</factor> - <factor name="FORGED_MUA_THEBAT_BOUN">2.00</factor> - <factor name="R_MISSING_CHARSET">5.00</factor> - <factor name="RCVD_DOUBLE_IP_SPAM">2.00</factor> - <factor name="OB_SURBL_MULTI">5.50</factor> - <factor name="FORGED_OUTLOOK_HTML">5.00</factor> - <factor name="HTML_MIME_NO_HTML_TAG">2.00</factor> - <factor name="R_BAD_EMAIL">10.50</factor> - <factor name="R_SPAM_FROM_LIBERO">10.00</factor> - <factor name="WHITELIST_IP">-2.00</factor> - <factor name="R_UNDISC_RCPT">5.00</factor> - <factor name="DRUGS_ANXIETY">2.00</factor> - <factor name="DRUGS_ANXIETY_EREC">2.00</factor> - <factor name="PH_SURBL_MULTI">5.50</factor> - <factor name="R_WHITE_ON_WHITE">9.00</factor> - <factor name="FAKE_HTML">1.00</factor> - <factor name="R_SPAM_FROM_VERSATEL">10.00</factor> - <factor name="HTML_SHORT_LINK_IMG_2">3.00</factor> - <factor name="FORGED_MUA_OUTLOOK">3.00</factor> - <factor name="R_FREE_HOSTING">4.00</factor> - <factor name="DRUGS_ERECTILE">2.00</factor> - <factor name="R_FREE_HOSTING_NAROD">3.00</factor> - <factor name="R_SPAM_FROM_ONO">10.00</factor> - <factor name="FM_FAKE_HELO_VERIZON">2.00</factor> - <factor name="REPTO_QUOTE_YAHOO">2.00</factor> - <factor name="MISSING_MIMEOLE">5.00</factor> - <factor name="RAMBLER_URIBL">0.50</factor> - <factor name="R_SPAM_FROM_VALUEHOST">10.00</factor> - <factor name="R_MIXED_CHARSET">5.00</factor> - <factor name="SORTED_RECIPS">3.50</factor> - <factor name="R_RCVD_SPAMBOTS">3.00</factor> - <factor name="JP_SURBL_MULTI">5.50</factor> - <factor name="R_TO_SEEMS_AUTO">3.00</factor> - <factor name="SUBJECT_NEEDS_ENCODING">1.00</factor> - <factor name="TRACKER_ID">3.84</factor> - <factor name="KAM_LOTTO1">7.00</factor> - <factor name="R_NO_SPACE_IN_FROM">3.00</factor> - <factor name="R_SAJDING">8.00</factor> - <factor name="R_BAD_CTE_7BIT">6.00</factor> - <factor name="WS_SURBL_MULTI">5.50</factor> - <factor name="R_POCHTA_RU">10.00</factor> - <factor name="R_FLASH_REDIR_IMGSHACK">10.00</factor> - <factor name="INVALID_MSGID">5.00</factor> - <factor name="R_FORGED_MPOP_WEBMAIL">8.00</factor> - <factor name="MISSING_MID">3.00</factor> - <factor name="DRUGS_DIET">2.00</factor> - <factor name="FORGED_RECIPIENTS">3.00</factor> - <factor name="RATWARE_MS_HASH">2.00</factor> - <factor name="HTML_TAG_BALANCE_HEAD">5.00</factor> - <factor name="STOX_REPLY_TYPE">1.00</factor> - <factor name="WINNOW_SPAM">1.00</factor> - <factor name="WINNOW_HAM">-1.00</factor> - <factor name="MIME_HEADER_CTYPE_ONLY">2.00</factor> - <factor name="R_FAKE_OUTLOOK">8.00</factor> -</factors> +<metric> + <name>default</name> + <required_score>10.0</required_score> + <symbol weight="8.00">R_SPAM_FROM_MTU</symbol> + <symbol weight="10.00">R_WWW_EKONF_COM</symbol> + <symbol weight="2.00">R_TINYURL</symbol> + <symbol weight="2.00">MISSING_SUBJECT</symbol> + <symbol weight="2.10">FORGED_OUTLOOK_TAGS</symbol> + <symbol weight="8.00">R_FAKE_THEBAT</symbol> + <symbol weight="5.00">FORGED_SENDER</symbol> + <symbol weight="2.00">DRUGS_MANYKINDS</symbol> + <symbol weight="3.30">ADVANCE_FEE_2</symbol> + <symbol weight="2.12">ADVANCE_FEE_3</symbol> + <symbol weight="3.50">SUSPICIOUS_RECIPS</symbol> + <symbol weight="6.00">FAKE_REPLY_C</symbol> + <symbol weight="1.00">MIME_HTML_ONLY</symbol> + <symbol weight="5.50">AB_SURBL_MULTI</symbol> + <symbol weight="2.00">FORGED_MSGID_YAHOO</symbol> + <symbol weight="5.50">SC_SURBL_MULTI</symbol> + <symbol weight="2.00">FORGED_MUA_THEBAT_BOUN</symbol> + <symbol weight="5.00">R_MISSING_CHARSET</symbol> + <symbol weight="2.00">RCVD_DOUBLE_IP_SPAM</symbol> + <symbol weight="5.50">OB_SURBL_MULTI</symbol> + <symbol weight="5.00">FORGED_OUTLOOK_HTML</symbol> + <symbol weight="2.00">HTML_MIME_NO_HTML_TAG</symbol> + <symbol weight="10.50">R_BAD_EMAIL</symbol> + <symbol weight="10.00">R_SPAM_FROM_LIBERO</symbol> + <symbol weight="-2.00">WHITELIST_IP</symbol> + <symbol weight="5.00">R_UNDISC_RCPT</symbol> + <symbol weight="2.00">DRUGS_ANXIETY</symbol> + <symbol weight="2.00">DRUGS_ANXIETY_EREC</symbol> + <symbol weight="5.50">PH_SURBL_MULTI</symbol> + <symbol weight="9.00">R_WHITE_ON_WHITE</symbol> + <symbol weight="1.00">FAKE_HTML</symbol> + <symbol weight="10.00">R_SPAM_FROM_VERSATEL</symbol> + <symbol weight="3.00">HTML_SHORT_LINK_IMG_2</symbol> + <symbol weight="3.00">FORGED_MUA_OUTLOOK</symbol> + <symbol weight="4.00">R_FREE_HOSTING</symbol> + <symbol weight="2.00">DRUGS_ERECTILE</symbol> + <symbol weight="3.00">R_FREE_HOSTING_NAROD</symbol> + <symbol weight="10.00">R_SPAM_FROM_ONO</symbol> + <symbol weight="2.00">FM_FAKE_HELO_VERIZON</symbol> + <symbol weight="2.00">REPTO_QUOTE_YAHOO</symbol> + <symbol weight="5.00">MISSING_MIMEOLE</symbol> + <symbol weight="0.50">RAMBLER_URIBL</symbol> + <symbol weight="10.00">R_SPAM_FROM_VALUEHOST</symbol> + <symbol weight="5.00">R_MIXED_CHARSET</symbol> + <symbol weight="3.50">SORTED_RECIPS</symbol> + <symbol weight="3.00">R_RCVD_SPAMBOTS</symbol> + <symbol weight="5.50">JP_SURBL_MULTI</symbol> + <symbol weight="3.00">R_TO_SEEMS_AUTO</symbol> + <symbol weight="1.00">SUBJECT_NEEDS_ENCODING</symbol> + <symbol weight="3.84">TRACKER_ID</symbol> + <symbol weight="7.00">KAM_LOTTO1</symbol> + <symbol weight="3.00">R_NO_SPACE_IN_FROM</symbol> + <symbol weight="8.00">R_SAJDING</symbol> + <symbol weight="6.00">R_BAD_CTE_7BIT</symbol> + <symbol weight="5.50">WS_SURBL_MULTI</symbol> + <symbol weight="10.00">R_POCHTA_RU</symbol> + <symbol weight="10.00">R_FLASH_REDIR_IMGSHACK</symbol> + <symbol weight="5.00">INVALID_MSGID</symbol> + <symbol weight="8.00">R_FORGED_MPOP_WEBMAIL</symbol> + <symbol weight="3.00">MISSING_MID</symbol> + <symbol weight="2.00">DRUGS_DIET</symbol> + <symbol weight="3.00">FORGED_RECIPIENTS</symbol> + <symbol weight="2.00">RATWARE_MS_HASH</symbol> + <symbol weight="5.00">HTML_TAG_BALANCE_HEAD</symbol> + <symbol weight="1.00">STOX_REPLY_TYPE</symbol> + <symbol weight="1.00">WINNOW_SPAM</symbol> + <symbol weight="-1.00">WINNOW_HAM</symbol> + <symbol weight="2.00">MIME_HEADER_CTYPE_ONLY</symbol> + <symbol weight="8.00">R_FAKE_OUTLOOK</symbol> +</metric> <!-- End of factors section --> <!-- Composites section --> diff --git a/src/cfg_file.h b/src/cfg_file.h index dec843359..e788be2e6 100644 --- a/src/cfg_file.h +++ b/src/cfg_file.h @@ -10,6 +10,7 @@ #include "mem_pool.h" #include "upstream.h" #include "memcached.h" +#include "symbols_cache.h" #define DEFAULT_BIND_PORT 768 #define DEFAULT_CONTROL_PORT 7608 @@ -278,33 +279,29 @@ struct config_file { GHashTable* variables; /**< hash of $variables defined in config, indexed by variable name */ GHashTable* metrics; /**< hash of metrics indexed by metric name */ GList* metrics_list; /**< linked list of metrics */ - GHashTable* factors; /**< hash of factors indexed by symbol name */ + GHashTable* metrics_symbols; /**< hash table of metrics indexed by symbol */ GHashTable* c_modules; /**< hash of c modules indexed by module name */ GHashTable* composite_symbols; /**< hash of composite symbols indexed by its name */ GList *classifiers; /**< list of all classifiers defined */ GList *statfiles; /**< list of all statfiles in config file order */ GHashTable *classifiers_symbols; /**< hashtable indexed by symbol name of classifiers */ GHashTable* cfg_params; /**< all cfg params indexed by its name in this structure */ - int clock_res; /**< resolution of clock used */ - double grow_factor; /**< grow factor for consolidation callback */ GList *views; /**< views */ GHashTable* domain_settings; /**< settings per-domains */ GHashTable* user_settings; /**< settings per-user */ gchar* domain_settings_str; /**< string representation of settings */ gchar* user_settings_str; + int clock_res; /**< resolution of clock used */ + + struct symbols_cache *cache; /**< symbols cache object */ + char *cache_filename; /**< filename of cache file */ + struct metric *default_metric; /**< default metric */ gchar* checksum; /**< real checksum of config file */ gchar* dump_checksum; /**< dump checksum of config file */ gpointer lua_state; /**< pointer to lua state */ }; -/** - * Add memcached server to config - * @param cf config file to use - * @param str line that describes server's credits - * @return 1 if line was successfully parsed and 0 in case of error - */ -int add_memcached_server (struct config_file *cf, gchar *str); /** * Parse host:port line @@ -397,15 +394,10 @@ void unescape_quotes (gchar *line); GList* parse_comma_list (memory_pool_t *pool, gchar *line); struct classifier_config* check_classifier_cfg (struct config_file *cfg, struct classifier_config *c); struct worker_conf* check_worker_conf (struct config_file *cfg, struct worker_conf *c); +struct metric* check_metric_conf (struct config_file *cfg, struct metric *c); gboolean parse_normalizer (struct config_file *cfg, struct statfile *st, const gchar *line); gboolean read_xml_config (struct config_file *cfg, const gchar *filename); -int yylex (void); -int yyparse (void); -void yyrestart (FILE *); -void parse_err (const gchar *fmt, ...); -void parse_warn (const gchar *fmt, ...); - #endif /* ifdef CFG_FILE_H */ /* * vi:ts=4 diff --git a/src/cfg_utils.c b/src/cfg_utils.c index 21cf57481..84115f4f7 100644 --- a/src/cfg_utils.c +++ b/src/cfg_utils.c @@ -40,35 +40,6 @@ #define DEFAULT_RLIMIT_NOFILE 2048 #define DEFAULT_RLIMIT_MAXCORE 0 -extern int yylineno; -extern gchar *yytext; - -int -add_memcached_server (struct config_file *cf, gchar *str) -{ - struct memcached_server *mc; - uint16_t port; - - if (str == NULL) - return 0; - - if (cf->memcached_servers_num == MAX_MEMCACHED_SERVERS) { - yywarn ("yyparse: maximum number of memcached servers is reached %d", MAX_MEMCACHED_SERVERS); - return 0; - } - - mc = &cf->memcached_servers[cf->memcached_servers_num]; - /* cur_tok - server name, str - server port */ - port = DEFAULT_MEMCACHED_PORT; - - if (!parse_host_port (str, &mc->addr, &port)) { - return 0; - } - - mc->port = port; - cf->memcached_servers_num++; - return 1; -} gboolean parse_host_port (const gchar *str, struct in_addr *ina, uint16_t *port) @@ -152,7 +123,7 @@ parse_bind_line (struct config_file *cfg, struct worker_conf *cf, gchar *str) if (stat (copy, &st) == -1) { if (errno == ENOENT) { if ((fd = open (str, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - yyerror ("parse_bind_line: cannot open path %s for making socket, %s", str, strerror (errno)); + msg_err ("cannot open path %s for making socket, %s", str, strerror (errno)); return 0; } else { @@ -161,13 +132,13 @@ parse_bind_line (struct config_file *cfg, struct worker_conf *cf, gchar *str) } } else { - yyerror ("parse_bind_line: cannot stat path %s for making socket, %s", str, strerror (errno)); + msg_err ("cannot stat path %s for making socket, %s", str, strerror (errno)); return 0; } } else { if (unlink (str) == -1) { - yyerror ("parse_bind_line: cannot remove path %s for making socket, %s", str, strerror (errno)); + msg_err ("cannot remove path %s for making socket, %s", str, strerror (errno)); return 0; } } @@ -199,15 +170,14 @@ init_defaults (struct config_file *cfg) cfg->max_statfile_size = DEFAULT_STATFILE_SIZE; - cfg->grow_factor = 1; cfg->modules_opts = g_hash_table_new (g_str_hash, g_str_equal); cfg->variables = g_hash_table_new (g_str_hash, g_str_equal); cfg->metrics = g_hash_table_new (g_str_hash, g_str_equal); - cfg->factors = g_hash_table_new (g_str_hash, g_str_equal); cfg->c_modules = g_hash_table_new (g_str_hash, g_str_equal); cfg->composite_symbols = g_hash_table_new (g_str_hash, g_str_equal); cfg->classifiers_symbols = g_hash_table_new (g_str_hash, g_str_equal); cfg->cfg_params = g_hash_table_new (g_str_hash, g_str_equal); + cfg->metrics_symbols = g_hash_table_new (g_str_hash, g_str_equal); init_settings (cfg); } @@ -221,14 +191,13 @@ free_config (struct config_file *cfg) g_hash_table_unref (cfg->variables); g_hash_table_remove_all (cfg->metrics); g_hash_table_unref (cfg->metrics); - g_hash_table_remove_all (cfg->factors); - g_hash_table_unref (cfg->factors); g_hash_table_remove_all (cfg->c_modules); g_hash_table_unref (cfg->c_modules); g_hash_table_remove_all (cfg->composite_symbols); g_hash_table_unref (cfg->composite_symbols); g_hash_table_remove_all (cfg->cfg_params); g_hash_table_unref (cfg->cfg_params); + g_hash_table_destroy (cfg->metrics_symbols); g_hash_table_destroy (cfg->classifiers_symbols); if (cfg->checksum) { g_free (cfg->checksum); @@ -387,7 +356,7 @@ substitute_variable (struct config_file *cfg, gchar *name, gchar *str, guchar re gboolean changed = FALSE; if (str == NULL) { - yywarn ("substitute_variable: trying to substitute variable in NULL string"); + msg_warn ("trying to substitute variable in NULL string"); return NULL; } @@ -405,7 +374,7 @@ substitute_variable (struct config_file *cfg, gchar *name, gchar *str, guchar re *v_end = '\0'; var = g_hash_table_lookup (cfg->variables, v_begin); if (var == NULL) { - yywarn ("substitute_variable: variable %s is not defined", v_begin); + msg_warn ("variable %s is not defined", v_begin); *v_end = t; p = v_end + 1; continue; @@ -587,23 +556,22 @@ post_load_config (struct config_file *cfg) cfg->clock_res = 3; } - if (g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC) == NULL) { - def_metric = memory_pool_alloc (cfg->cfg_pool, sizeof (struct metric)); + if ((def_metric = g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) { + def_metric = check_metric_conf (cfg, NULL); def_metric->name = DEFAULT_METRIC; - def_metric->func_name = "factors"; - def_metric->func = factor_consolidation_func; def_metric->required_score = DEFAULT_SCORE; def_metric->reject_score = DEFAULT_REJECT_SCORE; - def_metric->classifier = get_classifier ("winnow"); cfg->metrics_list = g_list_prepend (cfg->metrics_list, def_metric); g_hash_table_insert (cfg->metrics, DEFAULT_METRIC, def_metric); } + cfg->default_metric = def_metric; + /* Lua options */ (void)lua_post_load_config (cfg); } - +#if 0 void parse_err (const gchar *fmt, ...) { @@ -637,6 +605,7 @@ parse_warn (const gchar *fmt, ...) va_end (aq); g_warning ("%s", logbuf); } +#endif void unescape_quotes (gchar *line) @@ -697,6 +666,20 @@ check_classifier_cfg (struct config_file *cfg, struct classifier_config *c) return c; } +struct metric * +check_metric_conf (struct config_file *cfg, struct metric *c) +{ + if (c == NULL) { + c = memory_pool_alloc0 (cfg->cfg_pool, sizeof (struct metric)); + c->action = METRIC_ACTION_REJECT; + c->grow_factor = 1.0; + c->symbols = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func) g_hash_table_destroy, c->symbols); + } + + return c; +} + struct worker_conf * check_worker_conf (struct config_file *cfg, struct worker_conf *c) { diff --git a/src/cfg_xml.c b/src/cfg_xml.c index d4e977a3b..21c01e359 100644 --- a/src/cfg_xml.c +++ b/src/cfg_xml.c @@ -58,7 +58,6 @@ enum xml_config_section { XML_SECTION_METRIC, XML_SECTION_CLASSIFIER, XML_SECTION_STATFILE, - XML_SECTION_FACTORS, XML_SECTION_MODULE, XML_SECTION_MODULES, XML_SECTION_VIEW @@ -146,6 +145,12 @@ static struct xml_parser_rule grammar[] = { 0, NULL }, + { + "cache_file", + xml_handle_string, + G_STRUCT_OFFSET (struct config_file, cache_filename), + NULL + }, NULL_ATTR }, NULL_ATTR @@ -233,6 +238,12 @@ static struct xml_parser_rule grammar[] = { NULL }, { + "grow_factor", + xml_handle_double, + G_STRUCT_OFFSET (struct metric, grow_factor), + NULL + }, + { "required_score", xml_handle_double, G_STRUCT_OFFSET (struct metric, required_score), @@ -245,9 +256,9 @@ static struct xml_parser_rule grammar[] = { NULL }, { - "cache_file", - xml_handle_string, - G_STRUCT_OFFSET (struct metric, cache_filename), + "symbol", + handle_metric_symbol, + 0, NULL }, NULL_ATTR @@ -324,23 +335,6 @@ static struct xml_parser_rule grammar[] = { }, NULL_ATTR }, - { XML_SECTION_FACTORS, { - { - "grow_factor", - xml_handle_double, - G_STRUCT_OFFSET (struct config_file, grow_factor), - NULL - }, - { - "factor", - handle_factor, - 0, - NULL - }, - NULL_ATTR - }, - NULL_ATTR - }, { XML_SECTION_MODULE, { NULL_ATTR }, @@ -662,28 +656,39 @@ worker_handle_bind (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GH return TRUE; } -/* Factors section */ gboolean -handle_factor (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset) +handle_metric_symbol (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset) { - char *name, *err; + char *strval, *err; double *value; - - if ((name = g_hash_table_lookup (attrs, "name")) == NULL) { - msg_err ("factor tag must have \"name\" attribute"); - return FALSE; - } + GList *metric_list; + struct metric *metric = ctx->section_pointer; value = memory_pool_alloc (cfg->cfg_pool, sizeof (double)); - - errno = 0; - *value = strtod (data, &err); - if (errno != 0 || (err != NULL && *err != 0)) { - msg_err ("invalid number: %s, %s", data, strerror (errno)); - return FALSE; + if ((strval = g_hash_table_lookup (attrs, "weight")) == NULL) { + msg_info ("symbol tag should have \"weight\" attribute, assume weight 1.0"); + *value = 1.0; + } + else { + errno = 0; + *value = strtod (strval, &err); + if (errno != 0 || (err != NULL && *err != 0)) { + msg_err ("invalid number: %s, %s", strval, strerror (errno)); + return FALSE; + } } - g_hash_table_insert (cfg->factors, name, value); + g_hash_table_insert (metric->symbols, data, value); + + if ((metric_list = g_hash_table_lookup (cfg->metrics_symbols, data)) == NULL) { + metric_list = g_list_prepend (NULL, metric); + memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)g_list_free, metric_list); + g_hash_table_insert (cfg->metrics_symbols, data, metric_list); + } + else { + /* Slow but keep start element of list in safe */ + metric_list = g_list_append (metric_list, metric); + } return TRUE; } @@ -1198,9 +1203,6 @@ rspamd_xml_start_element (GMarkupParseContext *context, const gchar *element_nam ud->state = XML_ERROR; } } - else if (g_ascii_strcasecmp (element_name, "factors") == 0) { - ud->state = XML_READ_FACTORS; - } else if (g_ascii_strcasecmp (element_name, "modules") == 0) { ud->state = XML_READ_MODULES; } @@ -1208,16 +1210,9 @@ rspamd_xml_start_element (GMarkupParseContext *context, const gchar *element_nam ud->state = XML_READ_LOGGING; } else if (g_ascii_strcasecmp (element_name, "metric") == 0) { - if (extract_attr ("name", attribute_names, attribute_values, &res)) { - g_strlcpy (ud->section_name, res, sizeof (ud->section_name)); - ud->state = XML_READ_METRIC; - /* Create object */ - ud->section_pointer = memory_pool_alloc0 (ud->cfg->cfg_pool, sizeof (struct metric)); - } - else { - *error = g_error_new (xml_error_quark (), XML_PARAM_MISSING, "param 'name' is required for tag 'metric'"); - ud->state = XML_ERROR; - } + ud->state = XML_READ_METRIC; + /* Create object */ + ud->section_pointer = check_metric_conf (ud->cfg, NULL); } else if (g_ascii_strcasecmp (element_name, "classifier") == 0) { if (extract_attr ("type", attribute_names, attribute_values, &res)) { @@ -1271,7 +1266,6 @@ rspamd_xml_start_element (GMarkupParseContext *context, const gchar *element_nam case XML_READ_MODULE: case XML_READ_METRIC: case XML_READ_MODULES: - case XML_READ_FACTORS: case XML_READ_STATFILE: case XML_READ_WORKER: case XML_READ_LOGGING: @@ -1352,9 +1346,6 @@ rspamd_xml_end_element (GMarkupParseContext *context, const gchar *element_name, ud->state = XML_READ_CLASSIFIER; } break; - case XML_READ_FACTORS: - CHECK_TAG ("factors", FALSE); - break; case XML_READ_MODULES: CHECK_TAG ("modules", FALSE); break; @@ -1367,9 +1358,6 @@ rspamd_xml_end_element (GMarkupParseContext *context, const gchar *element_name, ud->state = XML_ERROR; return; } - if (m->classifier == NULL) { - m->classifier = get_classifier ("winnow"); - } g_hash_table_insert (ud->cfg->metrics, m->name, m); ud->cfg->metrics_list = g_list_prepend (ud->cfg->metrics_list, m); } @@ -1451,12 +1439,6 @@ rspamd_xml_text (GMarkupParseContext *context, const gchar *text, gsize text_len ud->state = XML_ERROR; } break; - case XML_READ_FACTORS: - if (!call_param_handler (ud, ud->section_name, val, cfg, XML_SECTION_FACTORS)) { - *error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "cannot parse tag '%s' data: %s", ud->section_name, val); - ud->state = XML_ERROR; - } - break; case XML_READ_METRIC: if (!call_param_handler (ud, ud->section_name, val, ud->section_pointer, XML_SECTION_METRIC)) { *error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "cannot parse tag '%s' data: %s", ud->section_name, val); @@ -1510,8 +1492,6 @@ xml_state_to_string (struct rspamd_xml_userdata *ud) return "read classifier section"; case XML_READ_STATFILE: return "read statfile section"; - case XML_READ_FACTORS: - return "read factors section"; case XML_READ_METRIC: return "read metric section"; case XML_READ_WORKER: @@ -1620,36 +1600,6 @@ xml_dump_variables (struct config_file *cfg, FILE *f) return TRUE; } -/* Dump factors section */ -static void -xml_factors_callback (gpointer key, gpointer value, gpointer user_data) -{ - FILE *f = user_data; - char *escaped_key; - - escaped_key = g_markup_escape_text (key, -1); - fprintf (f, " <factor name=\"%s\">%.2f</factor>" EOL, escaped_key, *(double *)value); - g_free (escaped_key); -} - -static gboolean -xml_dump_factors (struct config_file *cfg, FILE *f) -{ - /* Print header comment */ - fprintf (f, "<!-- Factors section -->" EOL "<factors>" EOL ); - - /* Iterate through variables */ - g_hash_table_foreach (cfg->factors, xml_factors_callback, (gpointer)f); - - /* Grow factor */ - fprintf (f, " <grow_factor>%.2f</grow_factor>" EOL, cfg->grow_factor); - - /* Print footer comment */ - fprintf (f, "</factors>" EOL "<!-- End of factors section -->" EOL EOL); - - return TRUE; -} - /* Composites section */ static void xml_composite_callback (gpointer key, gpointer value, gpointer user_data) @@ -2008,8 +1958,6 @@ xml_dump_config (struct config_file *cfg, const char *filename) CHECK_RES; res = xml_dump_variables (cfg, f); CHECK_RES; - res = xml_dump_factors (cfg, f); - CHECK_RES; res = xml_dump_composites (cfg, f); CHECK_RES; res = xml_dump_workers (cfg, f); diff --git a/src/cfg_xml.h b/src/cfg_xml.h index a4dc91b49..50fa7ac3f 100644 --- a/src/cfg_xml.h +++ b/src/cfg_xml.h @@ -19,7 +19,6 @@ enum xml_read_state { XML_READ_MODULES, XML_READ_CLASSIFIER, XML_READ_STATFILE, - XML_READ_FACTORS, XML_READ_METRIC, XML_READ_WORKER, XML_READ_VIEW, @@ -92,7 +91,7 @@ gboolean worker_handle_param (struct config_file *cfg, struct rspamd_xml_userdat gboolean worker_handle_type (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset); gboolean worker_handle_bind (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset); -gboolean handle_factor (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset); +gboolean handle_metric_symbol (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset); gboolean handle_module_opt (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset); diff --git a/src/classifiers/winnow.c b/src/classifiers/winnow.c index 9f66c3cbd..7599b1150 100644 --- a/src/classifiers/winnow.c +++ b/src/classifiers/winnow.c @@ -253,7 +253,7 @@ winnow_classify (struct classifier_ctx *ctx, statfile_pool_t * pool, GTree * inp sumbuf = memory_pool_alloc (task->task_pool, 32); snprintf (sumbuf, 32, "%.2Lg", max); cur = g_list_prepend (NULL, sumbuf); - insert_result (task, ctx->cfg->metric, sel->symbol, max, cur); + insert_result (task, sel->symbol, max, cur); } } diff --git a/src/filter.c b/src/filter.c index c6f936087..809d4b326 100644 --- a/src/filter.c +++ b/src/filter.c @@ -43,21 +43,14 @@ # include "lua/lua_common.h" #endif -void -insert_result (struct worker_task *task, const char *metric_name, const char *symbol, double flag, GList * opts) +static void +insert_metric_result (struct worker_task *task, struct metric *metric, const char *symbol, double flag, GList * opts) { - struct metric *metric; struct metric_result *metric_res; struct symbol *s; - struct cache_item *item; - GList *cur; - - metric = g_hash_table_lookup (task->worker->srv->cfg->metrics, metric_name); - if (metric == NULL) { - return; - } + gdouble *weight, w; - metric_res = g_hash_table_lookup (task->results, metric_name); + metric_res = g_hash_table_lookup (task->results, metric->name); if (metric_res == NULL) { /* Create new metric chain */ @@ -66,28 +59,48 @@ insert_result (struct worker_task *task, const char *metric_name, const char *sy metric_res->checked = FALSE; memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_hash_table_destroy, metric_res->symbols); metric_res->metric = metric; - g_hash_table_insert (task->results, (gpointer) metric_name, metric_res); + metric_res->grow_factor = 0; + g_hash_table_insert (task->results, (gpointer) metric->name, metric_res); + } + + weight = g_hash_table_lookup (metric->symbols, symbol); + if (weight == NULL) { + w = 1.0 * flag; + } + else { + w = (*weight) * flag; + } + /* Handle grow factor */ + if (metric_res->grow_factor && w > 0) { + w *= metric_res->grow_factor; + metric_res->grow_factor *= metric->grow_factor; + } + else if (w > 0) { + metric_res->grow_factor = metric->grow_factor; } + /* Add metric score */ + metric_res->score += w; + if ((s = g_hash_table_lookup (metric_res->symbols, symbol)) != NULL) { if (s->options && opts) { /* Append new options */ s->options = g_list_concat (s->options, opts); /* - * Note that there is no need to add new destructor of GList as elements of appended - * GList are used directly, so just free initial GList - */ + * Note that there is no need to add new destructor of GList as elements of appended + * GList are used directly, so just free initial GList + */ } else if (opts) { s->options = opts; memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_list_free, s->options); } - s->score = flag; + s->score = w; } else { s = memory_pool_alloc (task->task_pool, sizeof (struct symbol)); - s->score = flag; + s->score = w; s->options = opts; if (opts) { @@ -96,10 +109,35 @@ insert_result (struct worker_task *task, const char *metric_name, const char *sy g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s); } + debug_task ("got %.2f score for metric %s, factor: %f", s->score, metric->name, w); + +} + +void +insert_result (struct worker_task *task, const char *symbol, double flag, GList * opts) +{ + struct metric *metric; + struct cache_item *item; + GList *cur, *metric_list; + + metric_list = g_hash_table_lookup (task->cfg->metrics_symbols, symbol); + if (metric_list) { + cur = metric_list; + + while (cur) { + metric = cur->data; + insert_metric_result (task, metric, symbol, flag, opts); + cur = g_list_next (cur); + } + } + else { + /* Insert symbol to default metric */ + insert_metric_result (task, task->cfg->default_metric, symbol, flag, opts); + } /* Process cache item */ - if (metric->cache) { - cur = metric->cache->static_items; + if (task->cfg->cache) { + cur = task->cfg->cache->static_items; while (cur) { item = cur->data; @@ -109,7 +147,7 @@ insert_result (struct worker_task *task, const char *metric_name, const char *sy } cur = g_list_next (cur); } - cur = metric->cache->negative_items; + cur = task->cfg->cache->negative_items; while (cur) { item = cur->data; @@ -122,76 +160,6 @@ insert_result (struct worker_task *task, const char *metric_name, const char *sy } } -/* - * Default consolidation function based on factors in config file - */ -struct consolidation_callback_data { - struct worker_task *task; - double score; - int count; -}; - -static void -consolidation_callback (gpointer key, gpointer value, gpointer arg) -{ - double *factor, fs, grow = 1; - struct symbol *s = (struct symbol *)value; - struct consolidation_callback_data *data = (struct consolidation_callback_data *)arg; - struct worker_task *task = data->task; - - if (data->count > 0) { - grow = 1. + (data->task->cfg->grow_factor - 1.) * data->count; - } - - if (check_factor_settings (data->task, key, &fs)) { - if (s->score > 0) { - data->score += fs * s->score * grow; - data->count ++; - } - else { - data->score += fs * s->score; - } - } - else { - factor = g_hash_table_lookup (data->task->worker->srv->cfg->factors, key); - if (factor == NULL) { - debug_task ("got %.2f score for metric %s, factor: 1", s->score, (char *)key); - data->score += s->score; - } - else { - if (s->score > 0) { - data->score += *factor * s->score * grow; - data->count ++; - } - else { - data->score += *factor * s->score; - } - debug_task ("got %.2f score for metric %s, factor: %.2f", s->score, (char *)key, *factor); - } - } -} - -double -factor_consolidation_func (struct worker_task *task, const char *metric_name, const char *unused) -{ - struct metric_result *metric_res; - double res = 0.; - struct consolidation_callback_data data = { - .task = task, - .score = 0, - .count = 0 - }; - - metric_res = g_hash_table_lookup (task->results, metric_name); - if (metric_res == NULL) { - return res; - } - - g_hash_table_foreach (metric_res->symbols, consolidation_callback, &data); - - return data.score; -} - /* * Call perl or C module function for specified part of message */ @@ -203,7 +171,7 @@ call_filter_by_name (struct worker_task *task, const char *name, enum filter_typ switch (filt_type) { case C_FILTER: - c_module = g_hash_table_lookup (task->worker->srv->cfg->c_modules, name); + c_module = g_hash_table_lookup (task->cfg->c_modules, name); if (c_module) { res = 1; c_module->filter (task); @@ -227,41 +195,6 @@ call_filter_by_name (struct worker_task *task, const char *name, enum filter_typ debug_task ("filter name: %s, result: %d", name, (int)res); } -static void -metric_process_callback_common (gpointer key, gpointer value, void *data, gboolean is_forced) -{ - struct worker_task *task = (struct worker_task *)data; - struct metric_result *metric_res = (struct metric_result *)value; - - if (metric_res->checked && !is_forced) { - /* Already checked */ - return; - } - - /* Set flag */ - metric_res->checked = TRUE; - - if (metric_res->metric->func != NULL) { - metric_res->score = metric_res->metric->func (task, metric_res->metric->name, metric_res->metric->func_name); - } - else { - metric_res->score = factor_consolidation_func (task, metric_res->metric->name, NULL); - } - debug_task ("got result %.2f from consolidation function for metric %s", metric_res->score, metric_res->metric->name); -} - -static void -metric_process_callback_normal (gpointer key, gpointer value, void *data) -{ - metric_process_callback_common (key, value, data, FALSE); -} - -static void -metric_process_callback_forced (gpointer key, gpointer value, void *data) -{ - metric_process_callback_common (key, value, data, TRUE); -} - /* Return true if metric has score that is more than spam score for it */ static gboolean check_metric_is_spam (struct worker_task *task, struct metric *metric) @@ -271,7 +204,6 @@ check_metric_is_spam (struct worker_task *task, struct metric *metric) res = g_hash_table_lookup (task->results, metric->name); if (res) { - metric_process_callback_forced (metric->name, res, task); if (!check_metric_settings (task, metric, &ms, &rs)) { ms = metric->required_score; } @@ -284,29 +216,30 @@ check_metric_is_spam (struct worker_task *task, struct metric *metric) static int continue_process_filters (struct worker_task *task) { - GList *cur = task->save.entry; + GList *cur; gpointer item = task->save.item; + struct metric *metric; - struct metric *metric = cur->data; - - while (cur) { - metric = cur->data; - while (call_symbol_callback (task, metric->cache, &item)) { + while (call_symbol_callback (task, task->cfg->cache, &item)) { + cur = task->cfg->metrics_list; + while (cur) { + metric = cur->data; /* call_filter_by_name (task, filt->func_name, filt->type, SCRIPT_HEADER); */ if (task->save.saved) { task->save.entry = cur; task->save.item = item; return 0; } - else if (check_metric_is_spam (task, metric)) { - if (!task->pass_all_filters) { - break; - } + else if (!task->pass_all_filters && + metric->action == METRIC_ACTION_REJECT && + check_metric_is_spam (task, metric)) { + goto end; } + cur = g_list_next (cur); } - cur = g_list_next (cur); } +end: /* Process all statfiles */ process_statfiles (task); /* XXX: ugly direct call */ @@ -346,27 +279,25 @@ process_filters (struct worker_task *task) } /* Process metrics symbols */ - cur = task->worker->srv->cfg->metrics_list; - while (cur) { - metric = cur->data; - while (call_symbol_callback (task, metric->cache, &item)) { - /* call_filter_by_name (task, filt->func_name, filt->type, SCRIPT_HEADER); */ + while (call_symbol_callback (task, task->cfg->cache, &item)) { + /* Check reject actions */ + cur = task->cfg->metrics_list; + while (cur) { + metric = cur->data; if (task->save.saved) { task->save.entry = cur; task->save.item = item; return 0; } - else if (check_metric_is_spam (task, metric)) { - if (!task->pass_all_filters) { - break; - } + else if (!task->pass_all_filters && + metric->action == METRIC_ACTION_REJECT && + check_metric_is_spam (task, metric)) { + return 1; } + cur = g_list_next (cur); } - cur = g_list_next (cur); } - /* Process all metrics */ - g_hash_table_foreach (task->results, metric_process_callback_forced, task); return 1; } @@ -473,7 +404,6 @@ check_autolearn (struct statfile_autolearn_params *params, struct worker_task *t } else { /* Process score of metric */ - metric_process_callback_normal ((void *)metric_name, metric_res, task); if ((params->threshold_min != 0 && metric_res->score > params->threshold_min) || (params->threshold_max != 0 && metric_res->score < params->threshold_max)) { /* Now check for specific symbols */ if (params->symbols) { @@ -534,8 +464,6 @@ void make_composites (struct worker_task *task) { g_hash_table_foreach (task->results, composites_metric_callback, task); - /* Process all metrics */ - g_hash_table_foreach (task->results, metric_process_callback_forced, task); } diff --git a/src/filter.h b/src/filter.h index edc9523c6..24feaae9e 100644 --- a/src/filter.h +++ b/src/filter.h @@ -40,11 +40,17 @@ struct metric { char *name; /**< name of metric */ char *func_name; /**< name of consolidation function */ metric_cons_func func; /**< c consolidation function */ + double grow_factor; /**< grow factor for metric */ double required_score; /**< required score for this metric */ double reject_score; /**< reject score for this metric */ - struct classifier *classifier; /**< classifier that is used for metric */ - struct symbols_cache *cache; /**< symbols cache for metric */ - char *cache_filename; /**< filename of cache file */ + GHashTable *symbols; /**< weights of symbols in metric */ + enum { + METRIC_ACTION_REJECT = 0, + METRIC_ACTION_SOFT_REJECT, + METRIC_ACTION_REWRITE_SUBJECT, + METRIC_ACTION_ADD_HEADER, + METRIC_ACTION_GREYLIST + } action; /**< action to do by this metric */ }; /** @@ -55,6 +61,7 @@ struct metric_result { double score; /**< total score */ GHashTable *symbols; /**< symbols of metric */ gboolean checked; /**< whether metric result is consolidated */ + double grow_factor; /**< current grow factor */ }; /** @@ -78,7 +85,7 @@ void process_statfiles (struct worker_task *task); * @param flag numeric weight for symbol * @param opts list of symbol's options */ -void insert_result (struct worker_task *task, const char *metric_name, const char *symbol, double flag, GList *opts); +void insert_result (struct worker_task *task, const char *symbol, double flag, GList *opts); /** * Process all results and form composite metrics from existent metrics as it is defined in config diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c index d1a2b614b..aec72006c 100644 --- a/src/lua/lua_common.c +++ b/src/lua/lua_common.c @@ -191,7 +191,6 @@ init_lua (struct config_file *cfg) (void)luaopen_rspamd (L); (void)luaopen_logger (L); (void)luaopen_config (L); - (void)luaopen_metric (L); (void)luaopen_radix (L); (void)luaopen_hash_table (L); (void)luaopen_task (L); diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index bbbab4a53..5e0148ca7 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -31,33 +31,25 @@ /* Config file methods */ LUA_FUNCTION_DEF (config, get_module_opt); -LUA_FUNCTION_DEF (config, get_metric); LUA_FUNCTION_DEF (config, get_all_opt); LUA_FUNCTION_DEF (config, register_function); LUA_FUNCTION_DEF (config, add_radix_map); LUA_FUNCTION_DEF (config, add_hash_map); LUA_FUNCTION_DEF (config, get_classifier); +LUA_FUNCTION_DEF (config, register_symbol); static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF (config, get_module_opt), - LUA_INTERFACE_DEF (config, get_metric), LUA_INTERFACE_DEF (config, get_all_opt), LUA_INTERFACE_DEF (config, register_function), LUA_INTERFACE_DEF (config, add_radix_map), LUA_INTERFACE_DEF (config, add_hash_map), LUA_INTERFACE_DEF (config, get_classifier), + LUA_INTERFACE_DEF (config, register_symbol), {"__tostring", lua_class_tostring}, {NULL, NULL} }; -/* Metric methods */ -LUA_FUNCTION_DEF (metric, register_symbol); - -static const struct luaL_reg metriclib_m[] = { - LUA_INTERFACE_DEF (metric, register_symbol), - {"__tostring", lua_class_tostring}, - {NULL, NULL} -}; /* Radix tree */ LUA_FUNCTION_DEF (radix, get_key); @@ -85,14 +77,6 @@ lua_check_config (lua_State * L) return *((struct config_file **)ud); } -static struct metric * -lua_check_metric (lua_State * L) -{ - void *ud = luaL_checkudata (L, 1, "rspamd{metric}"); - luaL_argcheck (L, ud != NULL, 1, "'metric' expected"); - return *((struct metric **)ud); -} - static radix_tree_t * lua_check_radix (lua_State * L) { @@ -217,29 +201,6 @@ lua_config_get_all_opt (lua_State * L) static int -lua_config_get_metric (lua_State * L) -{ - struct config_file *cfg = lua_check_config (L); - struct metric *metric, **pmetric; - const char *name; - - if (cfg) { - name = luaL_checkstring (L, 2); - metric = g_hash_table_lookup (cfg->metrics, name); - if (metric) { - pmetric = lua_newuserdata (L, sizeof (struct metric *)); - lua_setclass (L, "rspamd{metric}", -1); - *pmetric = metric; - return 1; - } - } - - lua_pushnil (L); - return 1; - -} - -static int lua_config_get_classifier (lua_State * L) { struct config_file *cfg = lua_check_config (L); @@ -413,14 +374,14 @@ lua_metric_symbol_callback (struct worker_task *task, gpointer ud) } static int -lua_metric_register_symbol (lua_State * L) +lua_config_register_symbol (lua_State * L) { - struct metric *metric = lua_check_metric (L); + struct config_file *cfg = lua_check_config (L); const char *name, *callback; double weight; struct lua_callback_data *cd; - if (metric) { + if (cfg) { name = g_strdup (luaL_checkstring (L, 2)); weight = luaL_checknumber (L, 3); callback = luaL_checkstring (L, 4); @@ -428,7 +389,7 @@ lua_metric_register_symbol (lua_State * L) cd = g_malloc (sizeof (struct lua_callback_data)); cd->name = g_strdup (callback); cd->L = L; - register_symbol (&metric->cache, name, weight, lua_metric_symbol_callback, cd); + register_symbol (&cfg->cache, name, weight, lua_metric_symbol_callback, cd); } } return 1; @@ -483,15 +444,6 @@ luaopen_config (lua_State * L) } int -luaopen_metric (lua_State * L) -{ - lua_newclass (L, "rspamd{metric}", metriclib_m); - luaL_openlib (L, "rspamd_metric", null_reg, 0); - - return 1; -} - -int luaopen_radix (lua_State * L) { lua_newclass (L, "rspamd{radix}", radixlib_m); diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index dec415f9b..123cd048e 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -117,23 +117,22 @@ static int lua_task_insert_result (lua_State * L) { struct worker_task *task = lua_check_task (L); - const char *metric_name, *symbol_name, *param; + const char *symbol_name, *param; double flag; GList *params = NULL; int i, top; if (task != NULL) { - metric_name = memory_pool_strdup (task->task_pool, luaL_checkstring (L, 2)); - symbol_name = memory_pool_strdup (task->task_pool, luaL_checkstring (L, 3)); - flag = luaL_checknumber (L, 4); + symbol_name = memory_pool_strdup (task->task_pool, luaL_checkstring (L, 2)); + flag = luaL_checknumber (L, 3); top = lua_gettop (L); /* Get additional options */ - for (i = 5; i <= top; i++) { + for (i = 4; i <= top; i++) { param = luaL_checkstring (L, i); params = g_list_prepend (params, memory_pool_strdup (task->task_pool, param)); } - insert_result (task, metric_name, symbol_name, flag, params); + insert_result (task, symbol_name, flag, params); } return 1; } diff --git a/src/main.c b/src/main.c index b05315c1c..2b69afe38 100644 --- a/src/main.c +++ b/src/main.c @@ -54,7 +54,7 @@ rspamd_hash_t *counters; static struct rspamd_worker *fork_worker (struct rspamd_main *, struct worker_conf *); static gboolean load_rspamd_config (struct config_file *cfg, gboolean init_modules); -static void init_metrics_cache (struct config_file *cfg); +static void init_cfg_cache (struct config_file *cfg); sig_atomic_t do_restart; sig_atomic_t do_terminate; @@ -65,18 +65,12 @@ sig_atomic_t got_alarm; GQueue *signals_info; #endif -/* Yacc vars */ -extern int yynerrs; -extern FILE *yyin; -struct config_file *yacc_cfg; - static gboolean config_test; static gboolean no_fork; static gchar *cfg_name; static gchar *rspamd_user; static gchar *rspamd_group; static gchar *rspamd_pidfile; -static gchar *convert_config; static gboolean dump_vars; static gboolean dump_cache; @@ -97,7 +91,6 @@ static GOptionEntry entries[] = { "pid", 'p', 0, G_OPTION_ARG_STRING, &rspamd_pidfile, "Path to pidfile", NULL }, { "dump-vars", 'V', 0, G_OPTION_ARG_NONE, &dump_vars, "Print all rspamd variables and exit", NULL }, { "dump-cache", 'C', 0, G_OPTION_ARG_NONE, &dump_cache, "Dump symbols cache stats and exit", NULL }, - { "convert-config", 'X', 0, G_OPTION_ARG_STRING, &convert_config, "Convert old style of config to xml one", NULL }, { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL } }; @@ -290,7 +283,7 @@ reread_config (struct rspamd_main *rspamd) l = g_list_next (l); } init_lua_filters (rspamd->cfg); - init_metrics_cache (rspamd->cfg); + init_cfg_cache (rspamd->cfg); msg_info ("config rereaded successfully"); } } @@ -597,6 +590,8 @@ wait_for_workers (gpointer key, gpointer value, gpointer unused) return TRUE; } +#if 0 +/* XXX: remove this as it is unused now */ static gboolean convert_old_config (struct rspamd_main *rspamd) { @@ -638,6 +633,7 @@ convert_old_config (struct rspamd_main *rspamd) return FALSE; } +#endif static gboolean load_rspamd_config (struct config_file *cfg, gboolean init_modules) @@ -684,61 +680,47 @@ load_rspamd_config (struct config_file *cfg, gboolean init_modules) } static void -init_metrics_cache (struct config_file *cfg) +init_cfg_cache (struct config_file *cfg) { - struct metric *metric; - GList *l; - /* Init symbols cache for each metric */ - l = g_list_first (cfg->metrics_list); - while (l) { - metric = l->data; - if (metric->cache && !init_symbols_cache (cfg->cfg_pool, metric->cache, metric->cache_filename)) { - exit (EXIT_FAILURE); - } - l = g_list_next (l); + if (!init_symbols_cache (cfg->cfg_pool, cfg->cache, cfg->cache_filename)) { + exit (EXIT_FAILURE); } } static void -print_metrics_cache (struct config_file *cfg) +print_symbols_cache (struct config_file *cfg) { - struct metric *metric; - GList *l, *cur; + GList *cur; struct cache_item *item; int i; - l = g_list_first (cfg->metrics_list); - while (l) { - metric = l->data; - if (!init_symbols_cache (cfg->cfg_pool, metric->cache, metric->cache_filename)) { - exit (EXIT_FAILURE); - } - if (metric->cache) { - printf ("Cache for metric: %s\n", metric->name); + if (!init_symbols_cache (cfg->cfg_pool, cfg->cache, cfg->cache_filename)) { + exit (EXIT_FAILURE); + } + if (cfg->cache) { + printf ("Symbols cache\n"); + printf ("-----------------------------------------------------------------\n"); + printf ("| Pri | Symbol | Weight | Frequency | Avg. time |\n"); + i = 0; + cur = cfg->cache->negative_items; + while (cur) { + item = cur->data; printf ("-----------------------------------------------------------------\n"); - printf ("| Pri | Symbol | Weight | Frequency | Avg. time |\n"); - i = 0; - cur = metric->cache->negative_items; - while (cur) { - item = cur->data; - printf ("-----------------------------------------------------------------\n"); - printf ("| %3d | %22s | %6.1f | %9d | %9.3f |\n", i, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); - cur = g_list_next (cur); - i ++; - } - cur = metric->cache->static_items; - while (cur) { - item = cur->data; - printf ("-----------------------------------------------------------------\n"); - printf ("| %3d | %22s | %6.1f | %9d | %9.3f |\n", i, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); - cur = g_list_next (cur); - i ++; - } - + printf ("| %3d | %22s | %6.1f | %9d | %9.3f |\n", i, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); + cur = g_list_next (cur); + i ++; + } + cur = cfg->cache->static_items; + while (cur) { + item = cur->data; printf ("-----------------------------------------------------------------\n"); + printf ("| %3d | %22s | %6.1f | %9d | %9.3f |\n", i, item->s->symbol, item->s->weight, item->s->frequency, item->s->avg_time); + cur = g_list_next (cur); + i ++; } - l = g_list_next (l); + + printf ("-----------------------------------------------------------------\n"); } } @@ -816,12 +798,6 @@ main (int argc, char **argv, char **env) /* Init listen sockets hash */ listen_sockets = g_hash_table_new (g_direct_hash, g_direct_equal); - if (convert_config != NULL) { - if (! convert_old_config (rspamd)) { - exit (EXIT_FAILURE); - } - } - if (! load_rspamd_config (rspamd->cfg, TRUE)) { exit (EXIT_FAILURE); } @@ -847,7 +823,7 @@ main (int argc, char **argv, char **env) dump_cfg_vars (rspamd->cfg); } if (dump_cache) { - print_metrics_cache (rspamd->cfg); + print_symbols_cache (rspamd->cfg); exit (EXIT_SUCCESS); } fprintf (stderr, "syntax %s\n", res ? "OK" : "BAD"); @@ -905,8 +881,7 @@ main (int argc, char **argv, char **env) init_lua_filters (rspamd->cfg); - /* Init symbols cache for each metric */ - init_metrics_cache (rspamd->cfg); + init_cfg_cache (rspamd->cfg); flush_log_buf (); diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c index 8486f66c5..347fa5cab 100644 --- a/src/plugins/chartable.c +++ b/src/plugins/chartable.c @@ -26,7 +26,6 @@ * rspamd module that make marks based on symbol chains * * Allowed options: - * - metric (string): metric to insert symbol (default: 'default') * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET') * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters * (e.g. if threshold is 0.1 than charset change should occure more often than in 10 symbols), default: 0.1 @@ -45,7 +44,6 @@ struct chartable_ctx { int (*filter) (struct worker_task * task); - char *metric; char *symbol; double threshold; @@ -76,21 +74,7 @@ chartable_module_config (struct config_file *cfg) { char *value; int res = TRUE; - struct metric *metric; - double *w; - if ((value = get_module_opt (cfg, "chartable", "metric")) != NULL) { - chartable_module_ctx->metric = memory_pool_strdup (chartable_module_ctx->chartable_pool, value); - } - else { - chartable_module_ctx->metric = DEFAULT_METRIC; - } - if ((value = get_module_opt (cfg, "chartable", "symbol")) != NULL) { - chartable_module_ctx->symbol = memory_pool_strdup (chartable_module_ctx->chartable_pool, value); - } - else { - chartable_module_ctx->symbol = DEFAULT_SYMBOL; - } if ((value = get_module_opt (cfg, "chartable", "threshold")) != NULL) { errno = 0; chartable_module_ctx->threshold = strtod (value, NULL); @@ -103,20 +87,7 @@ chartable_module_config (struct config_file *cfg) chartable_module_ctx->threshold = DEFAULT_THRESHOLD; } - metric = g_hash_table_lookup (cfg->metrics, chartable_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", chartable_module_ctx->metric); - return FALSE; - } - - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, chartable_module_ctx->symbol); - if (w == NULL) { - register_symbol (&metric->cache, chartable_module_ctx->symbol, 1, chartable_symbol_callback, NULL); - } - else { - register_symbol (&metric->cache, chartable_module_ctx->symbol, *w, chartable_symbol_callback, NULL); - } + register_symbol (&cfg->cache, chartable_module_ctx->symbol, 1, chartable_symbol_callback, NULL); return res; } @@ -203,7 +174,7 @@ chartable_symbol_callback (struct worker_task *task, void *unused) while (cur) { part = cur->data; if (!part->is_empty && check_part (part, task->cfg->raw_mode)) { - insert_result (task, chartable_module_ctx->metric, chartable_module_ctx->symbol, 1, NULL); + insert_result (task, chartable_module_ctx->symbol, 1, NULL); } cur = g_list_next (cur); } diff --git a/src/plugins/emails.c b/src/plugins/emails.c index 37d0b8d33..ed6702396 100644 --- a/src/plugins/emails.c +++ b/src/plugins/emails.c @@ -26,7 +26,6 @@ * rspamd module that extracts emails from messages and check them via blacklist * * Allowed options: - * - metric (string): metric to insert symbol (default: 'default') * - symbol (string): symbol to insert (default: 'R_BAD_EMAIL') * - blacklist (map string): map that contains list of bad emails */ @@ -48,7 +47,6 @@ static const char *email_re_text = struct email_ctx { int (*filter) (struct worker_task * task); - char *metric; char *symbol; GRegex *email_re; @@ -89,15 +87,7 @@ emails_module_config (struct config_file *cfg) { char *value; int res = TRUE; - struct metric *metric; - double *w; - if ((value = get_module_opt (cfg, "emails", "metric")) != NULL) { - email_module_ctx->metric = memory_pool_strdup (email_module_ctx->email_pool, value); - } - else { - email_module_ctx->metric = DEFAULT_METRIC; - } if ((value = get_module_opt (cfg, "emails", "symbol")) != NULL) { email_module_ctx->symbol = memory_pool_strdup (email_module_ctx->email_pool, value); } @@ -110,20 +100,8 @@ emails_module_config (struct config_file *cfg) } } - metric = g_hash_table_lookup (cfg->metrics, email_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", email_module_ctx->metric); - return FALSE; - } - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, email_module_ctx->symbol); - if (w == NULL) { - register_symbol (&metric->cache, email_module_ctx->symbol, 1, emails_symbol_callback, NULL); - } - else { - register_symbol (&metric->cache, email_module_ctx->symbol, *w, emails_symbol_callback, NULL); - } + register_symbol (&cfg->cache, email_module_ctx->symbol, 1, emails_symbol_callback, NULL); return res; } @@ -233,7 +211,7 @@ emails_symbol_callback (struct worker_task *task, void *unused) while (cur) { if (g_hash_table_lookup (email_module_ctx->blacklist, cur->data) != NULL) { - insert_result (task, email_module_ctx->metric, email_module_ctx->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, (char *)cur->data))); + insert_result (task, email_module_ctx->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, (char *)cur->data))); } cur = g_list_next (cur); diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index ddfe696ab..3782e1194 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -26,7 +26,6 @@ * rspamd module that checks fuzzy checksums for messages * * Allowed options: - * - metric (string): metric to insert symbol (default: 'default') * - symbol (string): symbol to insert (default: 'R_FUZZY') * - max_score (double): maximum score to that weights of hashes would be normalized (default: 0 - no normalization) * @@ -74,7 +73,6 @@ struct fuzzy_mapping { struct fuzzy_ctx { int (*filter) (struct worker_task * task); - char *metric; char *symbol; struct storage_server *servers; int servers_num; @@ -260,15 +258,7 @@ fuzzy_check_module_config (struct config_file *cfg) { char *value; int res = TRUE; - struct metric *metric; - double *w; - if ((value = get_module_opt (cfg, "fuzzy_check", "metric")) != NULL) { - fuzzy_module_ctx->metric = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, value); - } - else { - fuzzy_module_ctx->metric = DEFAULT_METRIC; - } if ((value = get_module_opt (cfg, "fuzzy_check", "symbol")) != NULL) { fuzzy_module_ctx->symbol = memory_pool_strdup (fuzzy_module_ctx->fuzzy_pool, value); } @@ -306,31 +296,7 @@ fuzzy_check_module_config (struct config_file *cfg) parse_flags_string (value); } - metric = g_hash_table_lookup (cfg->metrics, fuzzy_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", fuzzy_module_ctx->metric); - return FALSE; - } - - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, fuzzy_module_ctx->symbol); - - if (w == NULL) { - if (fabs (fuzzy_module_ctx->max_score) < 0.001) { - register_symbol (&metric->cache, fuzzy_module_ctx->symbol, 1, fuzzy_symbol_callback, NULL); - } - else { - register_symbol (&metric->cache, fuzzy_module_ctx->symbol, fuzzy_module_ctx->max_score, fuzzy_symbol_callback, NULL); - } - } - else { - if (fabs (fuzzy_module_ctx->max_score) < 0.001) { - register_symbol (&metric->cache, fuzzy_module_ctx->symbol, *w, fuzzy_symbol_callback, NULL); - } - else { - register_symbol (&metric->cache, fuzzy_module_ctx->symbol, *w * fuzzy_module_ctx->max_score, fuzzy_symbol_callback, NULL); - } - } + register_symbol (&cfg->cache, fuzzy_module_ctx->symbol, fuzzy_module_ctx->max_score, fuzzy_symbol_callback, NULL); register_custom_controller_command ("fuzzy_add", fuzzy_add_handler, TRUE, TRUE); register_custom_controller_command ("fuzzy_del", fuzzy_delete_handler, TRUE, TRUE); @@ -418,7 +384,7 @@ fuzzy_io_callback (int fd, short what, void *arg) } snprintf (buf, sizeof (buf), "%d: %d / %.2f", flag, value, nval); - insert_result (session->task, fuzzy_module_ctx->metric, symbol, nval, g_list_prepend (NULL, + insert_result (session->task, symbol, nval, g_list_prepend (NULL, memory_pool_strdup (session->task->task_pool, buf))); } goto ok; diff --git a/src/plugins/lua/forged_recipients.lua b/src/plugins/lua/forged_recipients.lua index 77188e830..af4ce3125 100644 --- a/src/plugins/lua/forged_recipients.lua +++ b/src/plugins/lua/forged_recipients.lua @@ -1,7 +1,6 @@ -- Plugin for comparing smtp dialog recipients and sender with recipients and sender -- in mime headers -local metric = 'default' local symbol_rcpt = 'FORGED_RECIPIENTS' local symbol_sender = 'FORGED_SENDER' @@ -22,7 +21,7 @@ function check_forged_headers(task) end -- Check recipients count if count < table.maxn(smtp_rcpt) then - task:insert_result(metric, symbol_rcpt, 1) + task:insert_result(symbol_rcpt, 1) else -- Find pair for each smtp recipient recipient in To or Cc headers for _,sr in ipairs(smtp_rcpt) do @@ -48,7 +47,7 @@ function check_forged_headers(task) end if not res then - task:insert_result(metric, symbol_rcpt, 1) + task:insert_result(symbol_rcpt, 1) break end end @@ -59,7 +58,7 @@ function check_forged_headers(task) if smtp_form then local mime_from = msg:get_header('From') if not mime_from or not string.find(mime_from[0], smtp_from) then - task:insert_result(metric, symbol_sender, 1) + task:insert_result(symbol_sender, 1) end end end @@ -74,11 +73,7 @@ if opts then if opts['symbol_sender'] then symbol_sender = opts['symbol_sender'] end - if opts['metric'] then - metric = opts['metric'] - end - local m = rspamd_config:get_metric(metric) - m:register_symbol(symbol_rcpt, 1.0, 'check_forged_headers') + rspamd_config:register_symbol(symbol_rcpt, 1.0, 'check_forged_headers') end end diff --git a/src/plugins/lua/maillist.lua b/src/plugins/lua/maillist.lua index 18514aa86..098c512a4 100644 --- a/src/plugins/lua/maillist.lua +++ b/src/plugins/lua/maillist.lua @@ -1,7 +1,5 @@ -- Module for checking mail list headers - -local metric = 'default' local symbol = 'MAILLIST' -- EZMLM @@ -157,11 +155,11 @@ end function check_maillist(task) if check_ml_ezmlm(task) then - task:insert_result(metric, symbol, 1, 'ezmlm') + task:insert_result(symbol, 1, 'ezmlm') elseif check_ml_mailman(task) then - task:insert_result(metric, symbol, 1, 'mailman') + task:insert_result(symbol, 1, 'mailman') elseif check_ml_subscriberu(task) then - task:insert_result(metric, symbol, 1, 'subscribe.ru') + task:insert_result(symbol, 1, 'subscribe.ru') end end @@ -169,11 +167,7 @@ end local opts = rspamd_config:get_all_opt('maillist') if opts then if opts['symbol'] then - if opts['metric'] then - metric = opts['metric'] - end symbol = opts['symbol'] - local m = rspamd_config:get_metric(metric) - m:register_symbol(symbol, 1.0, 'check_maillist') + rspamd_config:register_symbol(symbol, 1.0, 'check_maillist') end end diff --git a/src/plugins/lua/once_received.lua b/src/plugins/lua/once_received.lua index d858314dd..404f13899 100644 --- a/src/plugins/lua/once_received.lua +++ b/src/plugins/lua/once_received.lua @@ -1,6 +1,5 @@ -- 0 or 1 received: = spam -local metric = 'default' local symbol = 'ONCE_RECEIVED' -- Symbol for strict checks local symbol_strict = nil @@ -10,7 +9,7 @@ local good_hosts = {} function check_quantity_received (task) local recvh = task:get_received_headers() if table.maxn(recvh) <= 1 then - task:insert_result(metric, symbol, 1) + task:insert_result(symbol, 1) -- Strict checks if symbol_strict then local r = recvh[1] @@ -19,7 +18,7 @@ function check_quantity_received (task) end -- Unresolved host if not r['real_hostname'] or string.lower(r['real_hostname']) == 'unknown' or string.match(r['real_hostname'], '^%d+%.%d+%.%d+%.%d+$') then - task:insert_result(metric, symbol_strict, 1) + task:insert_result(symbol_strict, 1) return end @@ -36,7 +35,7 @@ function check_quantity_received (task) end end if i then - task:insert_result(metric, symbol_strict, 1, h) + task:insert_result(symbol_strict, 1, h) return end end @@ -58,13 +57,10 @@ if opts then bad_hosts = v elseif n == 'good_host' then good_hosts = v - elseif n == 'metric' then - metric = v end end -- Register symbol's callback - local m = rspamd_config:get_metric(metric) - m:register_symbol(symbol, 1.0, 'check_quantity_received') + rspamd_config:register_symbol(symbol, 1.0, 'check_quantity_received') end end diff --git a/src/plugins/lua/received_rbl.lua b/src/plugins/lua/received_rbl.lua index 304678613..0a74272c1 100644 --- a/src/plugins/lua/received_rbl.lua +++ b/src/plugins/lua/received_rbl.lua @@ -3,11 +3,9 @@ -- .module 'received_rbl' { -- rbl = "insecure-bl.rambler.ru"; -- rbl = "xbl.spamhaus.org"; --- metric = "default"; -- symbol = "RECEIVED_RBL"; -- }; -local metric = 'default' local symbol = 'RECEIVED_RBL' local rbls = {} @@ -18,15 +16,15 @@ function dns_cb(task, to_resolve, results, err) -- Find incoming rbl in rbls list for _,rbl in ipairs(rbls) do if rbl == in_rbl then - task:insert_result(metric, symbol, 1, rbl .. ': ' .. ip) + task:insert_result(symbol, 1, rbl .. ': ' .. ip) else local s, _ = string.find(rbl, in_rbl) if s then s, _ = string.find(rbl, ':') if s then - task:insert_result(metric, string.sub(rbl, s + 1, -1), 1, ip) + task:insert_result(string.sub(rbl, s + 1, -1), 1, ip) else - task:insert_result(metric, symbol, 1, rbl .. ': ' .. ip) + task:insert_result(symbol, 1, rbl .. ': ' .. ip) end end end @@ -63,15 +61,11 @@ if opts then if opts['symbol'] then symbol = opts['symbol'] - if opts['metric'] then - metric = opts['metric'] - end if opts['rbl'] then rbls = opts['rbl'] end -- Register symbol's callback - local m = rspamd_config:get_metric(metric) - m:register_symbol(symbol, 1.0, 'received_cb') + rspamd_config:register_symbol(symbol, 1.0, 'received_cb') end -- If no symbol defined, do not register this module end diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua index e95aca6c7..34867bd2f 100644 --- a/src/plugins/lua/whitelist.lua +++ b/src/plugins/lua/whitelist.lua @@ -1,6 +1,5 @@ -- Module that add symbols to those hosts or from domains that are contained in whitelist -local metric = 'default' local symbol_ip = nil local symbol_from = nil @@ -14,7 +13,7 @@ function check_whitelist (task) if ipn then local key = r:get_key(ipn) if key then - task:insert_result(metric, symbol_ip, 1) + task:insert_result( symbol_ip, 1) end end end @@ -26,7 +25,7 @@ function check_whitelist (task) local _,_,domain = string.find(from, '@(.+)>?$') local key = h:get_key(domain) if key then - task:insert_result(metric, symbol_from, 1) + task:insert_result(symbol_from, 1) end end end @@ -58,16 +57,12 @@ if opts then end end - if opts['metric'] then - metric = opts['metric'] - end -- Register symbol's callback - local m = rspamd_config:get_metric(metric) if symbol_ip then - m:register_symbol(symbol_ip, 1.0, 'check_whitelist') + rspamd_config:register_symbol(symbol_ip, 1.0, 'check_whitelist') elseif symbol_from then - m:register_symbol(symbol_from, 1.0, 'check_whitelist') + rspamd_config:register_symbol(symbol_from, 1.0, 'check_whitelist') end end end diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c index 00fd7ad4c..2b52f4d8f 100644 --- a/src/plugins/regexp.c +++ b/src/plugins/regexp.c @@ -59,7 +59,6 @@ struct autolearn_data { struct regexp_ctx { int (*filter) (struct worker_task * task); GHashTable *autolearn_symbols; - char *metric; char *statfile_prefix; memory_pool_t *regexp_pool; @@ -87,7 +86,7 @@ regexp_dynamic_insert_result (struct worker_task *task, void *user_data) { char *symbol = user_data; - insert_result (task, regexp_module_ctx->metric, symbol, 1, NULL); + insert_result (task, symbol, 1, NULL); } static gboolean @@ -257,7 +256,6 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data) GList *cur_networks = NULL; struct dynamic_map_item *cur_nitem; memory_pool_t *new_pool; - struct metric *metric; if (data->prev_data) { jb = data->prev_data; @@ -296,13 +294,8 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data) } new_pool = memory_pool_new (memory_pool_get_size ()); - metric = g_hash_table_lookup (jb->cfg->metrics, regexp_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", regexp_module_ctx->metric); - return; - } - remove_dynamic_rules (metric->cache); + remove_dynamic_rules (jb->cfg->cache); if (regexp_module_ctx->dynamic_pool != NULL) { memory_pool_delete (regexp_module_ctx->dynamic_pool); } @@ -365,7 +358,7 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data) cur_item = memory_pool_alloc0 (new_pool, sizeof (struct regexp_module_item)); cur_item->symbol = cur_symbol; if (read_regexp_expression (new_pool, cur_item, cur_symbol, cur_rule, jb->cfg->raw_mode)) { - register_dynamic_symbol (new_pool, &metric->cache, cur_symbol, score, process_regexp_item, cur_item, cur_networks); + register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, process_regexp_item, cur_item, cur_networks); } else { msg_warn ("cannot parse dynamic rule"); @@ -373,7 +366,7 @@ json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data) } else { /* Just rule that is allways true (for whitelisting for example) */ - register_dynamic_symbol (new_pool, &metric->cache, cur_symbol, score, regexp_dynamic_insert_result, cur_symbol, cur_networks); + register_dynamic_symbol (new_pool, &jb->cfg->cache, cur_symbol, score, regexp_dynamic_insert_result, cur_symbol, cur_networks); } if (cur_networks) { g_list_free (cur_networks); @@ -439,18 +432,10 @@ regexp_module_config (struct config_file *cfg) GList *cur_opt = NULL; struct module_opt *cur; struct regexp_module_item *cur_item; - struct metric *metric; char *value; int res = TRUE; - double *w; struct regexp_json_buf *jb, **pjb; - if ((value = get_module_opt (cfg, "regexp", "metric")) != NULL) { - regexp_module_ctx->metric = memory_pool_strdup (regexp_module_ctx->regexp_pool, value); - } - else { - regexp_module_ctx->metric = DEFAULT_METRIC; - } if ((value = get_module_opt (cfg, "regexp", "statfile_prefix")) != NULL) { regexp_module_ctx->statfile_prefix = memory_pool_strdup (regexp_module_ctx->regexp_pool, value); } @@ -468,11 +453,6 @@ regexp_module_config (struct config_file *cfg) } } - metric = g_hash_table_lookup (cfg->metrics, regexp_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", regexp_module_ctx->metric); - return FALSE; - } cur_opt = g_hash_table_lookup (cfg->modules_opts, "regexp"); while (cur_opt) { @@ -515,14 +495,7 @@ regexp_module_config (struct config_file *cfg) break; } - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, cur->param); - if (w == NULL) { - register_symbol (&metric->cache, cur->param, 1, process_regexp_item, cur_item); - } - else { - register_symbol (&metric->cache, cur->param, *w, process_regexp_item, cur_item); - } + register_symbol (&cfg->cache, cur->param, 1, process_regexp_item, cur_item); cur_opt = g_list_next (cur_opt); } @@ -947,13 +920,13 @@ process_regexp_item (struct worker_task *task, void *user_data) if (item->lua_function) { /* Just call function */ if (lua_call_expression_func (item->lua_function, task, NULL, &res) && res) { - insert_result (task, regexp_module_ctx->metric, item->symbol, 1, NULL); + insert_result (task, item->symbol, 1, NULL); } } else { /* Process expression */ if (process_regexp_expression (item->expr, item->symbol, task, NULL)) { - insert_result (task, regexp_module_ctx->metric, item->symbol, 1, NULL); + insert_result (task, item->symbol, 1, NULL); } } } diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 8f1e8a07e..302d12cc6 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -26,7 +26,6 @@ * rspamd module that checks spf records of incoming email * * Allowed options: - * - metric (string): metric to insert symbol (default: 'default') * - symbol_allow (string): symbol to insert (default: 'R_SPF_ALLOW') * - symbol_fail (string): symbol to insert (default: 'R_SPF_FAIL') * - symbol_softfail (string): symbol to insert (default: 'R_SPF_SOFTFAIL') @@ -49,7 +48,6 @@ struct spf_ctx { int (*filter) (struct worker_task * task); - char *metric; char *symbol_fail; char *symbol_softfail; char *symbol_allow; @@ -79,15 +77,7 @@ spf_module_config (struct config_file *cfg) { char *value; int res = TRUE; - struct metric *metric; - double *w; - if ((value = get_module_opt (cfg, "spf", "metric")) != NULL) { - spf_module_ctx->metric = memory_pool_strdup (spf_module_ctx->spf_pool, value); - } - else { - spf_module_ctx->metric = DEFAULT_METRIC; - } if ((value = get_module_opt (cfg, "spf", "symbol_fail")) != NULL) { spf_module_ctx->symbol_fail = memory_pool_strdup (spf_module_ctx->spf_pool, value); } @@ -107,20 +97,7 @@ spf_module_config (struct config_file *cfg) spf_module_ctx->symbol_allow = DEFAULT_SYMBOL_ALLOW; } - metric = g_hash_table_lookup (cfg->metrics, spf_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", spf_module_ctx->metric); - return FALSE; - } - - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, spf_module_ctx->symbol_fail); - if (w == NULL) { - register_symbol (&metric->cache, spf_module_ctx->symbol_fail, 1, spf_symbol_callback, NULL); - } - else { - register_symbol (&metric->cache, spf_module_ctx->symbol_fail, *w, spf_symbol_callback, NULL); - } + register_symbol (&cfg->cache, spf_module_ctx->symbol_fail, 1, spf_symbol_callback, NULL); return res; } @@ -156,16 +133,16 @@ spf_plugin_callback (struct spf_record *record, struct worker_task *task) if ((s & m) == (addr->addr & m)) { switch (addr->mech) { case SPF_FAIL: - insert_result (task, spf_module_ctx->metric, spf_module_ctx->symbol_fail, 1, g_list_prepend (NULL, addr->spf_string)); + insert_result (task, spf_module_ctx->symbol_fail, 1, g_list_prepend (NULL, addr->spf_string)); task->messages = g_list_prepend (task->messages, "(SPF): spf fail"); break; case SPF_SOFT_FAIL: case SPF_NEUTRAL: - insert_result (task, spf_module_ctx->metric, spf_module_ctx->symbol_softfail, 1, g_list_prepend (NULL, addr->spf_string)); + insert_result (task, spf_module_ctx->symbol_softfail, 1, g_list_prepend (NULL, addr->spf_string)); task->messages = g_list_prepend (task->messages, "(SPF): spf softfail"); break; default: - insert_result (task, spf_module_ctx->metric, spf_module_ctx->symbol_allow, 1, g_list_prepend (NULL, addr->spf_string)); + insert_result (task, spf_module_ctx->symbol_allow, 1, g_list_prepend (NULL, addr->spf_string)); task->messages = g_list_prepend (task->messages, "(SPF): spf allow"); break; } diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 1f84e6946..6a5a83f3c 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -26,7 +26,6 @@ * rspamd module that implements SURBL url checking * * Allowed options: - * - metric (string): metric to insert symbol (default: 'default') * - weight (integer): weight of symbol * Redirecotor options: * - redirector (string): address of http redirector utility in format "host:port" @@ -116,8 +115,6 @@ surbl_module_config (struct config_file *cfg) struct module_opt *cur; struct suffix_item *new_suffix; struct surbl_bit_item *new_bit; - struct metric *metric; - double *w; char *value, *cur_tok, *str; uint32_t bit; @@ -177,12 +174,6 @@ surbl_module_config (struct config_file *cfg) else { surbl_module_ctx->max_urls = DEFAULT_SURBL_MAX_URLS; } - if ((value = get_module_opt (cfg, "surbl", "metric")) != NULL) { - surbl_module_ctx->metric = memory_pool_strdup (surbl_module_ctx->surbl_pool, value); - } - else { - surbl_module_ctx->metric = DEFAULT_METRIC; - } if ((value = get_module_opt (cfg, "surbl", "2tld")) != NULL) { if (add_map (value, read_host_list, fin_host_list, (void **)&surbl_module_ctx->tld2)) { surbl_module_ctx->tld2_file = memory_pool_strdup (surbl_module_ctx->surbl_pool, value + sizeof ("file://") - 1); @@ -194,12 +185,6 @@ surbl_module_config (struct config_file *cfg) } } - metric = g_hash_table_lookup (cfg->metrics, surbl_module_ctx->metric); - if (metric == NULL) { - msg_err ("cannot find metric definition %s", surbl_module_ctx->metric); - return FALSE; - } - cur_opt = g_hash_table_lookup (cfg->modules_opts, "surbl"); while (cur_opt) { @@ -213,14 +198,7 @@ surbl_module_config (struct config_file *cfg) msg_debug ("add new surbl suffix: %s with symbol: %s", new_suffix->suffix, new_suffix->symbol); *str = '_'; surbl_module_ctx->suffixes = g_list_prepend (surbl_module_ctx->suffixes, new_suffix); - /* Search in factors hash table */ - w = g_hash_table_lookup (cfg->factors, new_suffix->symbol); - if (w == NULL) { - register_symbol (&metric->cache, new_suffix->symbol, 1, surbl_test_url, new_suffix); - } - else { - register_symbol (&metric->cache, new_suffix->symbol, *w, surbl_test_url, new_suffix); - } + register_symbol (&cfg->cache, new_suffix->symbol, 1, surbl_test_url, new_suffix); } } if (!g_strncasecmp (cur->param, "bit", sizeof ("bit") - 1)) { @@ -244,13 +222,7 @@ surbl_module_config (struct config_file *cfg) new_suffix->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, DEFAULT_SURBL_SYMBOL); msg_debug ("add default surbl suffix: %s with symbol: %s", new_suffix->suffix, new_suffix->symbol); surbl_module_ctx->suffixes = g_list_prepend (surbl_module_ctx->suffixes, new_suffix); - w = g_hash_table_lookup (cfg->factors, new_suffix->symbol); - if (w == NULL) { - register_symbol (&metric->cache, new_suffix->symbol, 1, surbl_test_url, new_suffix); - } - else { - register_symbol (&metric->cache, new_suffix->symbol, *w, surbl_test_url, new_suffix); - } + register_symbol (&cfg->cache, new_suffix->symbol, 1, surbl_test_url, new_suffix); } return TRUE; @@ -495,18 +467,18 @@ process_dns_results (struct worker_task *task, struct suffix_item *suffix, char symbol = memory_pool_alloc (task->task_pool, len); snprintf (symbol, len, "%s%s%s", suffix->symbol, bit->symbol, c + 2); *c = '%'; - insert_result (task, surbl_module_ctx->metric, symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); + insert_result (task, symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); found = 1; } cur = g_list_next (cur); } if (!found) { - insert_result (task, surbl_module_ctx->metric, suffix->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); + insert_result (task, suffix->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); } } else { - insert_result (task, surbl_module_ctx->metric, suffix->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); + insert_result (task, suffix->symbol, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, url))); } } |