From dfbf2f58ee3f5f2481db498f006137eeb1dfdfb5 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 18 Jan 2016 01:07:11 +0000 Subject: [PATCH] Rework hyperscan notifies to reduce noise level --- src/hs_helper.c | 10 ++++--- src/libserver/re_cache.c | 48 ++++++++++++++++++++++++---------- src/libserver/re_cache.h | 9 ++++++- src/libserver/rspamd_control.c | 2 +- src/libserver/rspamd_control.h | 4 ++- src/worker.c | 12 ++++++--- 6 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/hs_helper.c b/src/hs_helper.c index bf317cbbb..771d74d7a 100644 --- a/src/hs_helper.c +++ b/src/hs_helper.c @@ -176,7 +176,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker, } if ((ncompiled = rspamd_re_cache_compile_hyperscan (ctx->cfg->re_cache, - ctx->hs_dir, ctx->max_time, + ctx->hs_dir, ctx->max_time, !forced, &err)) == -1) { msg_err ("failed to compile re cache: %e", err); g_error_free (err); @@ -184,8 +184,11 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker, return FALSE; } - msg_info ("compiled %d regular expressions to the hyperscan tree", - ncompiled); + if (ncompiled > 0) { + msg_info ("compiled %d regular expressions to the hyperscan tree", + ncompiled); + forced = TRUE; + } /* * Do not send notification unless all other workers are started @@ -198,6 +201,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker, srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED; srv_cmd.cmd.hs_loaded.cache_dir = ctx->hs_dir; + srv_cmd.cmd.hs_loaded.forced = forced; rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, NULL, NULL); diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 4434576c8..e0de03184 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -100,6 +100,7 @@ struct rspamd_re_cache { guint max_re_data; gchar hash[rspamd_cryptobox_HASHBYTES + 1]; #ifdef WITH_HYPERSCAN + gboolean hyperscan_loaded; gboolean disable_hyperscan; hs_platform_info_t plt; #endif @@ -186,11 +187,26 @@ rspamd_re_cache_new (void) cache->re_classes = g_hash_table_new (g_int64_hash, g_int64_equal); cache->nre = 0; cache->re = g_ptr_array_new_full (256, rspamd_re_cache_elt_dtor); +#ifdef WITH_HYPERSCAN + cache->hyperscan_loaded = FALSE; +#endif REF_INIT_RETAIN (cache, rspamd_re_cache_destroy); return cache; } +gboolean +rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache) +{ + g_assert (cache != NULL); + +#ifdef WITH_HYPERSCAN + return cache->hyperscan_loaded; +#else + return FALSE; +#endif +} + rspamd_regexp_t * rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re, enum rspamd_re_type type, gpointer type_data, gsize datalen) @@ -1011,7 +1027,7 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache, gint rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache, - const char *cache_dir, gdouble max_time, + const char *cache_dir, gdouble max_time, gboolean silent, GError **err) { g_assert (cache != NULL); @@ -1051,24 +1067,27 @@ rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache, g_assert (fd != -1); lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET); read (fd, &n, sizeof (n)); - total += n; close (fd); if (re_class->type_len > 0) { - msg_info_re_cache ( - "skip already valid class %s(%*s) to cache %6s, %d regexps", - rspamd_re_cache_type_to_string (re_class->type), - (gint) re_class->type_len - 1, - re_class->type_data, - re_class->hash, - n); + if (!silent) { + msg_info_re_cache ( + "skip already valid class %s(%*s) to cache %6s, %d regexps", + rspamd_re_cache_type_to_string (re_class->type), + (gint) re_class->type_len - 1, + re_class->type_data, + re_class->hash, + n); + } } else { - msg_info_re_cache ( - "skip already valid class %s to cache %6s, %d regexps", - rspamd_re_cache_type_to_string (re_class->type), - re_class->hash, - n); + if (!silent) { + msg_info_re_cache ( + "skip already valid class %s to cache %6s, %d regexps", + rspamd_re_cache_type_to_string (re_class->type), + re_class->hash, + n); + } } continue; @@ -1491,6 +1510,7 @@ rspamd_re_cache_load_hyperscan (struct rspamd_re_cache *cache, } msg_info_re_cache ("hyperscan database of %d regexps has been loaded", total); + cache->hyperscan_loaded = TRUE; return TRUE; #endif diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h index ae5cf9019..8358f146e 100644 --- a/src/libserver/re_cache.h +++ b/src/libserver/re_cache.h @@ -83,6 +83,13 @@ void rspamd_re_cache_replace (struct rspamd_re_cache *cache, void rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg); +/** + * Returns true when hyperscan is loaded + * @param cache + * @return + */ +gboolean rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache); + /** * Get runtime data for a cache */ @@ -145,7 +152,7 @@ enum rspamd_re_type rspamd_re_cache_type_from_string (const char *str); * Compile expressions to the hyperscan tree and store in the `cache_dir` */ gint rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache, - const char *cache_dir, gdouble max_time, + const char *cache_dir, gdouble max_time, gboolean silent, GError **err); diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c index 6696cd183..a91431cac 100644 --- a/src/libserver/rspamd_control.c +++ b/src/libserver/rspamd_control.c @@ -716,9 +716,9 @@ rspamd_srv_handler (gint fd, short what, gpointer ud) * workers */ wcmd.cmd.hs_loaded.cache_dir = cmd.cmd.hs_loaded.cache_dir; + wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced; rspamd_control_broadcast_cmd (srv, &wcmd, rspamd_control_hs_io_handler, NULL); - rdata->rep.reply.hs_loaded.unused = 0; break; default: msg_err ("unknown command type: %d", cmd.type); diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h index a15ace0d6..f8113cbb4 100644 --- a/src/libserver/rspamd_control.h +++ b/src/libserver/rspamd_control.h @@ -64,6 +64,7 @@ struct rspamd_control_command { } recompile; struct { gpointer cache_dir; + gboolean forced; } hs_loaded; struct { guint unused; @@ -112,6 +113,7 @@ struct rspamd_srv_command { } spair; struct { gpointer cache_dir; + gboolean forced; } hs_loaded; } cmd; }; @@ -124,7 +126,7 @@ struct rspamd_srv_reply { gint code; } spair; struct { - gint unused; + gint forced; } hs_loaded; } reply; }; diff --git a/src/worker.c b/src/worker.c index 9c5f0bd40..7598c687f 100644 --- a/src/worker.c +++ b/src/worker.c @@ -293,13 +293,19 @@ rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main, gpointer ud) { struct rspamd_control_reply rep; + struct rspamd_re_cache *cache = worker->srv->cfg->re_cache; - msg_info ("loading hyperscan expressions after receiving compilation notice"); memset (&rep, 0, sizeof (rep)); rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; - rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan ( - worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir); + if (!rspamd_re_cache_is_hs_loaded (cache) || cmd->cmd.hs_loaded.forced) { + msg_info ("loading hyperscan expressions after receiving compilation " + "notice: %s", + (!rspamd_re_cache_is_hs_loaded (cache)) ? + "new db" : "forced update"); + rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan ( + worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir); + } if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) { msg_err ("cannot write reply to the control socket: %s", -- 2.39.5