From ccea0a6fec04716e6ed162b314bf65f2f3205554 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 23 Apr 2018 17:31:59 +0100 Subject: [PATCH] [Project] Further fixes to maps usage --- src/libutil/map_helpers.c | 24 +++++++++++ src/plugins/dkim_check.c | 30 ++++++++------ src/plugins/fuzzy_check.c | 8 +++- src/plugins/surbl.c | 83 ++++++++++++++++++++++++++++++++++----- src/plugins/surbl.h | 73 ---------------------------------- 5 files changed, 122 insertions(+), 96 deletions(-) delete mode 100644 src/plugins/surbl.h diff --git a/src/libutil/map_helpers.c b/src/libutil/map_helpers.c index b22b430c6..2cc2a1efa 100644 --- a/src/libutil/map_helpers.c +++ b/src/libutil/map_helpers.c @@ -566,6 +566,10 @@ rspamd_map_helper_new_hash (struct rspamd_map *map) void rspamd_map_helper_destroy_hash (struct rspamd_hash_map_helper *r) { + if (r == NULL) { + return; + } + kh_destroy (rspamd_map_hash, r->htb); rspamd_mempool_delete (r->pool); } @@ -596,6 +600,10 @@ rspamd_map_helper_new_radix (struct rspamd_map *map) void rspamd_map_helper_destroy_radix (struct rspamd_radix_map_helper *r) { + if (r == NULL) { + return; + } + kh_destroy (rspamd_map_hash, r->htb); rspamd_mempool_delete (r->pool); } @@ -628,6 +636,10 @@ rspamd_map_helper_destroy_regexp (struct rspamd_regexp_map_helper *re_map) rspamd_regexp_t *re; guint i; + if (!re_map) { + return; + } + for (i = 0; i < re_map->regexps->len; i ++) { re = g_ptr_array_index (re_map->regexps, i); rspamd_regexp_unref (re); @@ -1094,6 +1106,10 @@ rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in) khiter_t k; struct rspamd_map_helper_value *val; + if (map == NULL) { + return NULL; + } + k = kh_get (rspamd_map_hash, map->htb, in); if (k != kh_end (map->htb)) { @@ -1112,6 +1128,10 @@ rspamd_match_radix_map (struct rspamd_radix_map_helper *map, { struct rspamd_map_helper_value *val; + if (map == NULL) { + return NULL; + } + val = (struct rspamd_map_helper_value *)radix_find_compressed (map->trie, in, inlen); @@ -1130,6 +1150,10 @@ rspamd_match_radix_map_addr (struct rspamd_radix_map_helper *map, { struct rspamd_map_helper_value *val; + if (map == NULL) { + return NULL; + } + val = (struct rspamd_map_helper_value *)radix_find_compressed_addr (map->trie, addr); if (val != (gconstpointer)RADIX_NO_VALUE) { diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 56e4bf700..d666289ec 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -35,6 +35,7 @@ #include "libserver/dkim.h" #include "libutil/hash.h" #include "libutil/map.h" +#include "libutil/map_helpers.h" #include "rspamd.h" #include "utlist.h" #include "lua/lua_common.h" @@ -65,8 +66,8 @@ struct dkim_ctx { const gchar *symbol_permfail; rspamd_mempool_t *dkim_pool; - radix_compressed_t *whitelist_ip; - GHashTable *dkim_domains; + struct rspamd_radix_map_helper *whitelist_ip; + struct rspamd_hash_map_helper *dkim_domains; guint strict_multiplier; guint time_jitter; rspamd_lru_hash_t *dkim_hash; @@ -311,7 +312,7 @@ dkim_module_config (struct rspamd_config *cfg) } lua_pop (cfg->lua_state, 1); /* Remove global function */ - dkim_module_ctx->whitelist_ip = radix_create_compressed (); + dkim_module_ctx->whitelist_ip = NULL; if ((value = rspamd_config_get_module_opt (cfg, "options", "check_local")) != NULL) { @@ -399,6 +400,10 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist", &dkim_module_ctx->whitelist_ip, NULL); + rspamd_mempool_add_destructor (dkim_module_ctx->dkim_pool, + (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_radix, + dkim_module_ctx->whitelist_ip); + } if ((value = @@ -410,6 +415,9 @@ dkim_module_config (struct rspamd_config *cfg) ucl_object_tostring (value)); } else { + rspamd_mempool_add_destructor (dkim_module_ctx->dkim_pool, + (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_hash, + dkim_module_ctx->dkim_domains); got_trusted = TRUE; } } @@ -423,6 +431,9 @@ dkim_module_config (struct rspamd_config *cfg) ucl_object_tostring (value)); } else { + rspamd_mempool_add_destructor (dkim_module_ctx->dkim_pool, + (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_hash, + dkim_module_ctx->dkim_domains); got_trusted = TRUE; } } @@ -797,11 +808,6 @@ dkim_module_reconfig (struct rspamd_config *cfg) saved_ctx = dkim_module_ctx->ctx; rspamd_mempool_delete (dkim_module_ctx->dkim_pool); - radix_destroy_compressed (dkim_module_ctx->whitelist_ip); - - if (dkim_module_ctx->dkim_domains) { - g_hash_table_destroy (dkim_module_ctx->dkim_domains); - } if (dkim_module_ctx->dkim_hash) { rspamd_lru_hash_destroy (dkim_module_ctx->dkim_hash); @@ -864,7 +870,7 @@ dkim_module_check (struct dkim_check_result *res) if (dkim_module_ctx->dkim_domains != NULL) { /* Perform strict check */ if ((strict_value = - g_hash_table_lookup (dkim_module_ctx->dkim_domains, + rspamd_match_hash_map (dkim_module_ctx->dkim_domains, rspamd_dkim_get_domain (cur->ctx))) != NULL) { if (!dkim_module_parse_strict (strict_value, &cur->mult_allow, &cur->mult_deny)) { @@ -1020,8 +1026,8 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) return; } /* Check whitelist */ - if (radix_find_compressed_addr (dkim_module_ctx->whitelist_ip, - task->from_addr) != RADIX_NO_VALUE) { + if (rspamd_match_radix_map_addr (dkim_module_ctx->whitelist_ip, + task->from_addr) != NULL) { msg_info_task ("skip DKIM checks for whitelisted address"); return; } @@ -1085,7 +1091,7 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || - g_hash_table_lookup (dkim_module_ctx->dkim_domains, + rspamd_match_hash_map (dkim_module_ctx->dkim_domains, rspamd_dkim_get_domain (ctx)) == NULL)) { msg_debug_task ("skip dkim check for %s domain", rspamd_dkim_get_domain (ctx)); diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 90f04e059..fe95dc715 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -33,6 +33,7 @@ #include "config.h" #include "libmime/message.h" #include "libutil/map.h" +#include "libutil/map_helpers.h" #include "libmime/images.h" #include "libserver/worker_util.h" #include "fuzzy_wire.h" @@ -88,7 +89,7 @@ struct fuzzy_rule { gboolean fuzzy_images; gboolean short_text_direct_hash; gint learn_condition_cb; - GHashTable *skip_map; + struct rspamd_hash_map_helper *skip_map; }; struct fuzzy_ctx { @@ -425,6 +426,9 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rspamd_map_add_from_ucl (cfg, value, "Fuzzy hashes whitelist", rspamd_kv_list_read, rspamd_kv_list_fin, (void **)&rule->skip_map); + rspamd_mempool_add_destructor (fuzzy_module_ctx->fuzzy_pool, + (rspamd_mempool_destruct_t)rspamd_map_helper_destroy_radix, + rule->skip_map); } else { rule->skip_map = NULL; @@ -1911,7 +1915,7 @@ fuzzy_insert_result (struct fuzzy_client_session *session, rspamd_encode_hex_buf (cmd->digest, sizeof (cmd->digest), hexbuf, sizeof (hexbuf) - 1); hexbuf[sizeof (hexbuf) - 1] = '\0'; - if (g_hash_table_lookup (session->rule->skip_map, hexbuf)) { + if (rspamd_match_hash_map (session->rule->skip_map, hexbuf)) { return; } } diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c index 01cfb5944..62b96e75f 100644 --- a/src/plugins/surbl.c +++ b/src/plugins/surbl.c @@ -35,9 +35,11 @@ #include "config.h" #include "libmime/message.h" #include "libutil/map.h" +#include "libutil/map_helpers.h" #include "rspamd.h" -#include "surbl.h" #include "utlist.h" +#include "multipattern.h" +#include "monitored.h" #include "libserver/html.h" #include "libutil/http_private.h" #include "unix-std.h" @@ -62,6 +64,71 @@ INIT_LOG_MODULE(surbl) +#define DEFAULT_SURBL_WEIGHT 10 +#define DEFAULT_REDIRECTOR_READ_TIMEOUT 5.0 +#define DEFAULT_SURBL_SYMBOL "SURBL_DNS" +#define SURBL_OPTION_NOIP (1 << 0) +#define SURBL_OPTION_RESOLVEIP (1 << 1) +#define SURBL_OPTION_CHECKIMAGES (1 << 2) +#define MAX_LEVELS 10 + +struct surbl_ctx { + struct module_ctx ctx; + guint16 weight; + gdouble read_timeout; + gboolean use_tags; + GList *suffixes; + gchar *metric; + const gchar *redirector_symbol; + GHashTable **exceptions; + struct rspamd_hash_map_helper *whitelist; + void *redirector_map_data; + GHashTable *redirector_tlds; + guint use_redirector; + guint max_redirected_urls; + gint redirector_cbid; + struct upstream_list *redirectors; + rspamd_mempool_t *surbl_pool; +}; + +struct suffix_item { + guint64 magic; + const gchar *monitored_domain; + const gchar *suffix; + const gchar *symbol; + guint32 options; + GArray *bits; + GHashTable *ips; + struct rspamd_monitored *m; + gint callback_id; + gint url_process_cbref; +}; + +struct dns_param { + struct rspamd_url *url; + struct rspamd_task *task; + gchar *host_resolve; + struct suffix_item *suffix; + struct rspamd_async_watcher *w; +}; + +struct redirector_param { + struct rspamd_url *url; + struct rspamd_task *task; + struct upstream *redirector; + struct rspamd_http_connection *conn; + GHashTable *tree; + struct suffix_item *suffix; + struct rspamd_async_watcher *w; + gint sock; + guint redirector_requests; +}; + +struct surbl_bit_item { + guint32 bit; + gchar *symbol; +}; + #define SURBL_REDIRECTOR_CALLBACK "SURBL_REDIRECTOR_CALLBACK" static struct surbl_ctx *surbl_module_ctx = NULL; @@ -304,10 +371,9 @@ surbl_module_init (struct rspamd_config *cfg, struct module_ctx **ctx) surbl_module_ctx->surbl_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); surbl_module_ctx->redirectors = NULL; - surbl_module_ctx->whitelist = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); + surbl_module_ctx->whitelist = NULL; rspamd_mempool_add_destructor (surbl_module_ctx->surbl_pool, - (rspamd_mempool_destruct_t) g_hash_table_destroy, + (rspamd_mempool_destruct_t) rspamd_map_helper_destroy_hash, surbl_module_ctx->whitelist); surbl_module_ctx->exceptions = rspamd_mempool_alloc0 ( surbl_module_ctx->surbl_pool, MAX_LEVELS * sizeof (GHashTable *)); @@ -901,7 +967,7 @@ surbl_module_config (struct rspamd_config *cfg) if ((value = rspamd_config_get_module_opt (cfg, "surbl", "whitelist")) != NULL) { rspamd_map_add_from_ucl (cfg, value, - "SURBL whitelist", rspamd_hosts_read, rspamd_hosts_fin, + "SURBL whitelist", rspamd_kv_list_read, rspamd_kv_list_fin, (void **)&surbl_module_ctx->whitelist); } @@ -979,15 +1045,14 @@ surbl_module_reconfig (struct rspamd_config *cfg) surbl_module_ctx->surbl_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), NULL); surbl_module_ctx->redirectors = NULL; - surbl_module_ctx->whitelist = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); + surbl_module_ctx->whitelist = NULL; /* Zero exceptions hashes */ surbl_module_ctx->exceptions = rspamd_mempool_alloc0 ( surbl_module_ctx->surbl_pool, MAX_LEVELS * sizeof (GHashTable *)); /* Register destructors */ rspamd_mempool_add_destructor (surbl_module_ctx->surbl_pool, - (rspamd_mempool_destruct_t) g_hash_table_destroy, + (rspamd_mempool_destruct_t) rspamd_map_helper_destroy_hash, surbl_module_ctx->whitelist); rspamd_mempool_add_destructor (surbl_module_ctx->surbl_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, @@ -1136,7 +1201,7 @@ format_surbl_request (rspamd_mempool_t * pool, url->surbllen = r; if (!forced && - g_hash_table_lookup (surbl_module_ctx->whitelist, result) != NULL) { + rspamd_match_hash_map (surbl_module_ctx->whitelist, result) != NULL) { msg_debug_pool ("url %s is whitelisted", result); g_set_error (err, SURBL_ERROR, WHITELIST_ERROR, diff --git a/src/plugins/surbl.h b/src/plugins/surbl.h deleted file mode 100644 index 9671ac6c1..000000000 --- a/src/plugins/surbl.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef RSPAMD_MODULE_SURBL -#define RSPAMD_MODULE_SURBL - -#include "config.h" -#include "multipattern.h" -#include "monitored.h" - -#define DEFAULT_SURBL_WEIGHT 10 -#define DEFAULT_REDIRECTOR_READ_TIMEOUT 5.0 -#define DEFAULT_SURBL_SYMBOL "SURBL_DNS" -#define SURBL_OPTION_NOIP (1 << 0) -#define SURBL_OPTION_RESOLVEIP (1 << 1) -#define SURBL_OPTION_CHECKIMAGES (1 << 2) -#define MAX_LEVELS 10 - -struct surbl_ctx { - struct module_ctx ctx; - guint16 weight; - gdouble read_timeout; - gboolean use_tags; - GList *suffixes; - gchar *metric; - const gchar *redirector_symbol; - GHashTable **exceptions; - GHashTable *whitelist; - void *redirector_map_data; - GHashTable *redirector_tlds; - guint use_redirector; - guint max_redirected_urls; - gint redirector_cbid; - struct upstream_list *redirectors; - rspamd_mempool_t *surbl_pool; -}; - -struct suffix_item { - guint64 magic; - const gchar *monitored_domain; - const gchar *suffix; - const gchar *symbol; - guint32 options; - GArray *bits; - GHashTable *ips; - struct rspamd_monitored *m; - gint callback_id; - gint url_process_cbref; -}; - -struct dns_param { - struct rspamd_url *url; - struct rspamd_task *task; - gchar *host_resolve; - struct suffix_item *suffix; - struct rspamd_async_watcher *w; -}; - -struct redirector_param { - struct rspamd_url *url; - struct rspamd_task *task; - struct upstream *redirector; - struct rspamd_http_connection *conn; - GHashTable *tree; - struct suffix_item *suffix; - struct rspamd_async_watcher *w; - gint sock; - guint redirector_requests; -}; - -struct surbl_bit_item { - guint32 bit; - gchar *symbol; -}; - -#endif -- 2.39.5