diff options
author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2019-11-11 15:30:12 +0000 |
---|---|---|
committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2019-11-11 15:30:30 +0000 |
commit | 3c4d37339225ea645f0385aeee1b86238ec41f7c (patch) | |
tree | f3bf673a02d9ead6eac7fe5ffeebf5e924282bac | |
parent | 63800059e8dacc1ba69b623719c21355a77301c5 (diff) | |
download | rspamd-3c4d37339225ea645f0385aeee1b86238ec41f7c.tar.gz rspamd-3c4d37339225ea645f0385aeee1b86238ec41f7c.zip |
[Feature] Allow to limit maps per specific worker
-rw-r--r-- | src/controller.c | 4 | ||||
-rw-r--r-- | src/fuzzy_storage.c | 16 | ||||
-rw-r--r-- | src/libserver/cfg_file.h | 3 | ||||
-rw-r--r-- | src/libserver/cfg_utils.c | 24 | ||||
-rw-r--r-- | src/libserver/dynamic_cfg.c | 2 | ||||
-rw-r--r-- | src/libserver/worker_util.c | 5 | ||||
-rw-r--r-- | src/libutil/logger.c | 4 | ||||
-rw-r--r-- | src/libutil/map.c | 36 | ||||
-rw-r--r-- | src/libutil/map.h | 16 | ||||
-rw-r--r-- | src/libutil/util.c | 4 | ||||
-rw-r--r-- | src/lua/lua_map.c | 39 | ||||
-rw-r--r-- | src/plugins/dkim_check.c | 8 | ||||
-rw-r--r-- | src/plugins/fuzzy_check.c | 7 | ||||
-rw-r--r-- | src/plugins/spf.c | 2 | ||||
-rw-r--r-- | src/rspamd_proxy.c | 10 | ||||
-rw-r--r-- | src/worker.c | 2 |
16 files changed, 129 insertions, 53 deletions
diff --git a/src/controller.c b/src/controller.c index 1eb088d90..2ab796791 100644 --- a/src/controller.c +++ b/src/controller.c @@ -3513,7 +3513,9 @@ start_controller_worker (struct rspamd_worker *worker) if (ctx->secure_ip != NULL) { rspamd_config_radix_from_ucl (ctx->cfg, ctx->secure_ip, "Allow unauthenticated requests from these addresses", - &ctx->secure_map, NULL); + &ctx->secure_map, + NULL, + worker); } ctx->lang_det = ctx->cfg->lang_det; diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c index 94bbc6484..04acc6ba5 100644 --- a/src/fuzzy_storage.c +++ b/src/fuzzy_storage.c @@ -1975,7 +1975,7 @@ start_fuzzy (struct rspamd_worker *worker) if (ctx->update_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->update_map, "Allow fuzzy updates from specified addresses", - &ctx->update_ips, NULL); + &ctx->update_ips, NULL, worker); } if (ctx->skip_map != NULL) { @@ -1986,7 +1986,8 @@ start_fuzzy (struct rspamd_worker *worker) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&ctx->skip_hashes)) == NULL) { + (void **)&ctx->skip_hashes, + worker)) == NULL) { msg_warn_config ("cannot load hashes list from %s", ucl_object_tostring (ctx->skip_map)); } @@ -1998,14 +1999,18 @@ start_fuzzy (struct rspamd_worker *worker) if (ctx->blocked_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->blocked_map, "Block fuzzy requests from the specific IPs", - &ctx->blocked_ips, NULL); + &ctx->blocked_ips, + NULL, + worker); } /* Create radix trees */ if (ctx->ratelimit_whitelist_map != NULL) { rspamd_config_radix_from_ucl (worker->srv->cfg, ctx->ratelimit_whitelist_map, "Skip ratelimits from specific ip addresses/networks", - &ctx->ratelimit_whitelist, NULL); + &ctx->ratelimit_whitelist, + NULL, + worker); } /* Ratelimits */ @@ -2019,7 +2024,8 @@ start_fuzzy (struct rspamd_worker *worker) ctx->resolver = rspamd_dns_resolver_init (worker->srv->logger, ctx->event_loop, worker->srv->cfg); - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, worker, 0); + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, + ctx->resolver, worker, RSPAMD_MAP_WATCH_WORKER); /* Get peer pipe */ memset (&srv_cmd, 0, sizeof (srv_cmd)); diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index e604fe43a..fbddd9096 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -758,7 +758,8 @@ gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, const ucl_object_t *obj, const gchar *description, struct rspamd_radix_map_helper **target, - GError **err); + GError **err, + struct rspamd_worker *worker); /** * Adds new settings id to be preprocessed diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index ba009cc81..857f7a0ac 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -1149,7 +1149,8 @@ rspamd_include_map_handler (const guchar *data, gsize len, rspamd_ucl_read_cb, rspamd_ucl_fin_cb, rspamd_ucl_dtor_cb, - (void **)pcbdata) != NULL; + (void **)pcbdata, + NULL) != NULL; } /* @@ -2180,10 +2181,11 @@ rspamd_config_get_action_by_type (struct rspamd_config *cfg, gboolean rspamd_config_radix_from_ucl (struct rspamd_config *cfg, - const ucl_object_t *obj, - const gchar *description, - struct rspamd_radix_map_helper **target, - GError **err) + const ucl_object_t *obj, + const gchar *description, + struct rspamd_radix_map_helper **target, + GError **err, + struct rspamd_worker *worker) { ucl_type_t type; ucl_object_iter_t it = NULL; @@ -2207,8 +2209,10 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)target) == NULL) { - g_set_error (err, g_quark_from_static_string ("rspamd-config"), + (void **)target, + worker) == NULL) { + g_set_error (err, + g_quark_from_static_string ("rspamd-config"), EINVAL, "bad map definition %s for %s", str, ucl_object_key (obj)); return FALSE; @@ -2232,8 +2236,10 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)target) == NULL) { - g_set_error (err, g_quark_from_static_string ("rspamd-config"), + (void **)target, + worker) == NULL) { + g_set_error (err, + g_quark_from_static_string ("rspamd-config"), EINVAL, "bad map object for %s", ucl_object_key (obj)); return FALSE; } diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c index 45c6838ec..a39778ec2 100644 --- a/src/libserver/dynamic_cfg.c +++ b/src/libserver/dynamic_cfg.c @@ -283,7 +283,7 @@ init_dynamic_config (struct rspamd_config *cfg) json_config_read_cb, json_config_fin_cb, json_config_dtor_cb, - (void **)pjb)) { + (void **)pjb, NULL)) { msg_err ("cannot add map for configuration %s", cfg->dynamic_conf); } } diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index 08933060c..362d64bc5 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -1955,7 +1955,8 @@ rspamd_worker_init_controller (struct rspamd_worker *worker, ev_timer_start (ctx->event_loop, &cbd.save_stats_event); rspamd_map_watch (worker->srv->cfg, ctx->event_loop, - ctx->resolver, worker, TRUE); + ctx->resolver, worker, + RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER); if (prrd != NULL) { if (ctx->cfg->rrd_file && worker->index == 0) { @@ -1992,6 +1993,6 @@ rspamd_worker_init_controller (struct rspamd_worker *worker, } else { rspamd_map_watch (worker->srv->cfg, ctx->event_loop, - ctx->resolver, worker, FALSE); + ctx->resolver, worker, RSPAMD_MAP_WATCH_SCANNER); } }
\ No newline at end of file diff --git a/src/libutil/logger.c b/src/libutil/logger.c index 31d018533..193dd4fce 100644 --- a/src/libutil/logger.c +++ b/src/libutil/logger.c @@ -579,7 +579,9 @@ rspamd_set_logger (struct rspamd_config *cfg, rspamd_config_radix_from_ucl (cfg, cfg->debug_ip_map, "IP addresses for which debug logs are enabled", - &logger->debug_ip, NULL); + &logger->debug_ip, + NULL, + NULL); } else if (logger->debug_ip) { rspamd_map_helper_destroy_radix (logger->debug_ip); diff --git a/src/libutil/map.c b/src/libutil/map.c index d8c990a76..6fe2a80ca 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -1985,22 +1985,40 @@ rspamd_map_watch (struct rspamd_config *cfg, struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, - gboolean active_http) + enum rspamd_map_watch_type how) { GList *cur = cfg->maps; struct rspamd_map *map; struct rspamd_map_backend *bk; guint i; + g_assert (how > RSPAMD_MAP_WATCH_MIN && how < RSPAMD_MAP_WATCH_MAX); + /* First of all do synced read of data */ while (cur) { map = cur->data; map->event_loop = event_loop; map->r = resolver; - map->wrk = worker; - if (active_http) { - map->active_http = active_http; + if (map->wrk == NULL && how != RSPAMD_MAP_WATCH_WORKER) { + /* Generic scanner map */ + map->wrk = worker; + + if (how == RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER) { + map->active_http = TRUE; + } + else { + map->active_http = FALSE; + } + } + else if (map->wrk != NULL && map->wrk == worker) { + /* Map is bound to a specific worker */ + map->active_http = TRUE; + } + else { + /* Skip map for this worker as irrelevant */ + cur = g_list_next (cur); + continue; } if (!map->active_http) { @@ -2590,7 +2608,8 @@ rspamd_map_add (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data) + void **user_data, + struct rspamd_worker *worker) { struct rspamd_map *map; struct rspamd_map_backend *bk; @@ -2617,6 +2636,7 @@ rspamd_map_add (struct rspamd_config *cfg, map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->backends = g_ptr_array_sized_new (1); + map->wrk = worker; rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, map->backends); g_ptr_array_add (map->backends, bk); @@ -2663,7 +2683,8 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data) + void **user_data, + struct rspamd_worker *worker) { ucl_object_iter_t it = NULL; const ucl_object_t *cur, *elt; @@ -2676,7 +2697,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, if (ucl_object_type (obj) == UCL_STRING) { /* Just a plain string */ return rspamd_map_add (cfg, ucl_object_tostring (obj), description, - read_callback, fin_callback, dtor, user_data); + read_callback, fin_callback, dtor, user_data, worker); } map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map)); @@ -2689,6 +2710,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); map->backends = g_ptr_array_new (); + map->wrk = worker; rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, map->backends); map->poll_timeout = cfg->map_timeout; diff --git a/src/libutil/map.h b/src/libutil/map.h index 9e09ab8fe..ce49bacbb 100644 --- a/src/libutil/map.h +++ b/src/libutil/map.h @@ -70,7 +70,8 @@ struct rspamd_map *rspamd_map_add (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data); + void **user_data, + struct rspamd_worker *worker); /** * Add map from ucl @@ -81,7 +82,16 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg, map_cb_t read_callback, map_fin_cb_t fin_callback, map_dtor_t dtor, - void **user_data); + void **user_data, + struct rspamd_worker *worker); + +enum rspamd_map_watch_type { + RSPAMD_MAP_WATCH_MIN = 9, + RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER, + RSPAMD_MAP_WATCH_SCANNER, + RSPAMD_MAP_WATCH_WORKER, + RSPAMD_MAP_WATCH_MAX +}; /** * Start watching of maps by adding events to libevent event loop @@ -90,7 +100,7 @@ void rspamd_map_watch (struct rspamd_config *cfg, struct ev_loop *event_loop, struct rspamd_dns_resolver *resolver, struct rspamd_worker *worker, - gboolean active_http); + enum rspamd_map_watch_type how); /** * Preloads maps where all backends are file diff --git a/src/libutil/util.c b/src/libutil/util.c index 5ada3a27e..264101376 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -2432,7 +2432,9 @@ rspamd_config_libs (struct rspamd_external_libs_ctx *ctx, if (cfg->local_addrs) { rspamd_config_radix_from_ucl (cfg, cfg->local_addrs, "Local addresses", - ctx->local_addrs, NULL); + ctx->local_addrs, + NULL, + NULL); } if (cfg->ssl_ca_path) { diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index bead7ae4a..13674e6b1 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -161,7 +161,8 @@ lua_config_add_radix_map (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_warn_config ("invalid radix map %s", map_line); lua_pushnil (L); @@ -218,7 +219,8 @@ lua_config_radix_from_config (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_err_config ("invalid radix map static"); lua_pushnil (L); ucl_object_unref (fake_obj); @@ -279,7 +281,8 @@ lua_config_radix_from_ucl (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { msg_err_config ("invalid radix map static"); lua_pushnil (L); ucl_object_unref (fake_obj); @@ -324,7 +327,8 @@ lua_config_add_hash_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { msg_warn_config ("invalid set map %s", map_line); lua_pushnil (L); return 1; @@ -364,7 +368,8 @@ lua_config_add_kv_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { msg_warn_config ("invalid hash map %s", map_line); lua_pushnil (L); @@ -529,7 +534,8 @@ lua_config_add_map (lua_State *L) lua_map_read, lua_map_fin, lua_map_dtor, - (void **)&map->data.cbdata)) == NULL) { + (void **)&map->data.cbdata, + NULL)) == NULL) { if (cbidx != -1) { luaL_unref (L, LUA_REGISTRYINDEX, cbidx); @@ -554,7 +560,8 @@ lua_config_add_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -571,7 +578,8 @@ lua_config_add_map (lua_State *L) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&map->data.hash)) == NULL) { + (void **)&map->data.hash, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -588,7 +596,8 @@ lua_config_add_map (lua_State *L) rspamd_radix_read, rspamd_radix_fin, rspamd_radix_dtor, - (void **)&map->data.radix)) == NULL) { + (void **)&map->data.radix, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -605,7 +614,8 @@ lua_config_add_map (lua_State *L) rspamd_regexp_list_read_single, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -622,7 +632,8 @@ lua_config_add_map (lua_State *L) rspamd_regexp_list_read_multiple, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -639,7 +650,8 @@ lua_config_add_map (lua_State *L) rspamd_glob_list_read_single, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); @@ -656,7 +668,8 @@ lua_config_add_map (lua_State *L) rspamd_glob_list_read_multiple, rspamd_regexp_list_fin, rspamd_regexp_list_dtor, - (void **) &map->data.re_map)) == NULL) { + (void **) &map->data.re_map, + NULL)) == NULL) { lua_pushnil (L); ucl_object_unref (map_obj); diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index ebe2292a1..3a88d9bb6 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -435,7 +435,7 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "dkim", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "DKIM whitelist", - &dkim_module_ctx->whitelist_ip, NULL); + &dkim_module_ctx->whitelist_ip, NULL, NULL); } if ((value = @@ -445,7 +445,8 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&dkim_module_ctx->dkim_domains)) { + (void **)&dkim_module_ctx->dkim_domains, + NULL)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_object_tostring (value)); } @@ -461,7 +462,8 @@ dkim_module_config (struct rspamd_config *cfg) rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&dkim_module_ctx->dkim_domains)) { + (void **)&dkim_module_ctx->dkim_domains, + NULL)) { msg_warn_config ("cannot load dkim domains list from %s", ucl_object_tostring (value)); } diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 3162c1f0e..5c3994559 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -359,7 +359,8 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, rspamd_kv_list_read, rspamd_kv_list_fin, rspamd_kv_list_dtor, - (void **)&rule->skip_map); + (void **)&rule->skip_map, + NULL); } if ((value = ucl_object_lookup (obj, "headers")) != NULL) { @@ -1042,7 +1043,9 @@ fuzzy_check_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "fuzzy_check", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "Fuzzy whitelist", - &fuzzy_module_ctx->whitelist, NULL); + &fuzzy_module_ctx->whitelist, + NULL, + NULL); } else { fuzzy_module_ctx->whitelist = NULL; diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 6f120786c..cc52fbd0b 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -335,7 +335,7 @@ spf_module_config (struct rspamd_config *cfg) rspamd_config_get_module_opt (cfg, "spf", "whitelist")) != NULL) { rspamd_config_radix_from_ucl (cfg, value, "SPF whitelist", - &spf_module_ctx->whitelist_ip, NULL); + &spf_module_ctx->whitelist_ip, NULL, NULL); } cb_id = rspamd_symcache_add_symbol (cfg->cache, diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 737fff608..ae51e3e5b 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -2297,8 +2297,14 @@ start_rspamd_proxy (struct rspamd_worker *worker) rspamd_worker_init_controller (worker, NULL); } else { - rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, - worker, 0); + if (ctx->has_self_scan) { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, + worker, RSPAMD_MAP_WATCH_SCANNER); + } + else { + rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, + worker, RSPAMD_MAP_WATCH_WORKER); + } } rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, diff --git a/src/worker.c b/src/worker.c index 4279aa82c..b20800a60 100644 --- a/src/worker.c +++ b/src/worker.c @@ -552,7 +552,7 @@ start_worker (struct rspamd_worker *worker) } else { rspamd_map_watch (worker->srv->cfg, ctx->event_loop, ctx->resolver, - worker, 0); + worker, RSPAMD_MAP_WATCH_SCANNER); } rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->event_loop, |