Rework hyperscan notifies to reduce noise level

diff --git a/src/hs_helper.c b/src/hs_helper.c
index bf317cbbbac63be7d95046b163c02965f02274d7..771d74d7a8619a68cb3ee35b989581899d2b77f2 100644 (file)
--- a/src/hs_helper.c
+++ b/src/hs_helper.c
@@ -176,7 +176,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
        }
 
        if ((ncompiled = rspamd_re_cache_compile_hyperscan (ctx->cfg->re_cache,
-                       ctx->hs_dir, ctx->max_time,
+                       ctx->hs_dir, ctx->max_time, !forced,
                        &err)) == -1) {
                msg_err ("failed to compile re cache: %e", err);
                g_error_free (err);
@@ -184,8 +184,11 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
                return FALSE;
        }
 
-       msg_info ("compiled %d regular expressions to the hyperscan tree",
-                       ncompiled);
+       if (ncompiled > 0) {
+               msg_info ("compiled %d regular expressions to the hyperscan tree",
+                               ncompiled);
+               forced = TRUE;
+       }
 
        /*
         * Do not send notification unless all other workers are started
@@ -198,6 +201,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
 
        srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED;
        srv_cmd.cmd.hs_loaded.cache_dir = ctx->hs_dir;
+       srv_cmd.cmd.hs_loaded.forced = forced;
 
        rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, NULL, NULL);
 

diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 4434576c819efc8f135bb191041322d38e67b59c..e0de031849900eeb92b8686d4353de2d438e58d0 100644 (file)
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -100,6 +100,7 @@ struct rspamd_re_cache {
        guint max_re_data;
        gchar hash[rspamd_cryptobox_HASHBYTES + 1];
 #ifdef WITH_HYPERSCAN
+       gboolean hyperscan_loaded;
        gboolean disable_hyperscan;
        hs_platform_info_t plt;
 #endif
@@ -186,11 +187,26 @@ rspamd_re_cache_new (void)
        cache->re_classes = g_hash_table_new (g_int64_hash, g_int64_equal);
        cache->nre = 0;
        cache->re = g_ptr_array_new_full (256, rspamd_re_cache_elt_dtor);
+#ifdef WITH_HYPERSCAN
+       cache->hyperscan_loaded = FALSE;
+#endif
        REF_INIT_RETAIN (cache, rspamd_re_cache_destroy);
 
        return cache;
 }
 
+gboolean
+rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache)
+{
+       g_assert (cache != NULL);
+
+#ifdef WITH_HYPERSCAN
+       return cache->hyperscan_loaded;
+#else
+       return FALSE;
+#endif
+}
+
 rspamd_regexp_t *
 rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re,
                enum rspamd_re_type type, gpointer type_data, gsize datalen)
@@ -1011,7 +1027,7 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache,
 
 gint
 rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
-               const char *cache_dir, gdouble max_time,
+               const char *cache_dir, gdouble max_time, gboolean silent,
                GError **err)
 {
        g_assert (cache != NULL);
@@ -1051,24 +1067,27 @@ rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
                        g_assert (fd != -1);
                        lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET);
                        read (fd, &n, sizeof (n));
-                       total += n;
                        close (fd);
 
                        if (re_class->type_len > 0) {
-                               msg_info_re_cache (
-                                               "skip already valid class %s(%*s) to cache %6s, %d regexps",
-                                               rspamd_re_cache_type_to_string (re_class->type),
-                                               (gint) re_class->type_len - 1,
-                                               re_class->type_data,
-                                               re_class->hash,
-                                               n);
+                               if (!silent) {
+                                       msg_info_re_cache (
+                                                       "skip already valid class %s(%*s) to cache %6s, %d regexps",
+                                                       rspamd_re_cache_type_to_string (re_class->type),
+                                                       (gint) re_class->type_len - 1,
+                                                       re_class->type_data,
+                                                       re_class->hash,
+                                                       n);
+                               }
                        }
                        else {
-                               msg_info_re_cache (
-                                               "skip already valid class %s to cache %6s, %d regexps",
-                                               rspamd_re_cache_type_to_string (re_class->type),
-                                               re_class->hash,
-                                               n);
+                               if (!silent) {
+                                       msg_info_re_cache (
+                                                       "skip already valid class %s to cache %6s, %d regexps",
+                                                       rspamd_re_cache_type_to_string (re_class->type),
+                                                       re_class->hash,
+                                                       n);
+                               }
                        }
 
                        continue;
@@ -1491,6 +1510,7 @@ rspamd_re_cache_load_hyperscan (struct rspamd_re_cache *cache,
        }
 
        msg_info_re_cache ("hyperscan database of %d regexps has been loaded", total);
+       cache->hyperscan_loaded = TRUE;
 
        return TRUE;
 #endif

diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h
index ae5cf901999b73f0319aba0b3021e8805f116553..8358f146e183409aa215509da132046dc0ef576b 100644 (file)
--- a/src/libserver/re_cache.h
+++ b/src/libserver/re_cache.h
@@ -83,6 +83,13 @@ void rspamd_re_cache_replace (struct rspamd_re_cache *cache,
 void rspamd_re_cache_init (struct rspamd_re_cache *cache,
                struct rspamd_config *cfg);
 
+/**
+ * Returns true when hyperscan is loaded
+ * @param cache
+ * @return
+ */
+gboolean rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache);
+
 /**
  * Get runtime data for a cache
  */
@@ -145,7 +152,7 @@ enum rspamd_re_type rspamd_re_cache_type_from_string (const char *str);
  * Compile expressions to the hyperscan tree and store in the `cache_dir`
  */
 gint rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
-               const char *cache_dir, gdouble max_time,
+               const char *cache_dir, gdouble max_time, gboolean silent,
                GError **err);
 
 

diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
index 6696cd183dafe6ac06e7ae2e3dfdabcfa00c88e5..a91431cac416bc9421ab615fe25cb16236ddcd80 100644 (file)
--- a/src/libserver/rspamd_control.c
+++ b/src/libserver/rspamd_control.c
@@ -716,9 +716,9 @@ rspamd_srv_handler (gint fd, short what, gpointer ud)
                                 * workers
                                 */
                                wcmd.cmd.hs_loaded.cache_dir = cmd.cmd.hs_loaded.cache_dir;
+                               wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced;
                                rspamd_control_broadcast_cmd (srv, &wcmd,
                                                rspamd_control_hs_io_handler, NULL);
-                               rdata->rep.reply.hs_loaded.unused = 0;
                                break;
                        default:
                                msg_err ("unknown command type: %d", cmd.type);

diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h
index a15ace0d649c26be315e5076554a1693ad9ae039..f8113cbb448eecceae36af58e2ae3e8b2016dd99 100644 (file)
--- a/src/libserver/rspamd_control.h
+++ b/src/libserver/rspamd_control.h
@@ -64,6 +64,7 @@ struct rspamd_control_command {
                } recompile;
                struct {
                        gpointer cache_dir;
+                       gboolean forced;
                } hs_loaded;
                struct {
                        guint unused;
@@ -112,6 +113,7 @@ struct rspamd_srv_command {
                } spair;
                struct {
                        gpointer cache_dir;
+                       gboolean forced;
                } hs_loaded;
        } cmd;
 };
@@ -124,7 +126,7 @@ struct rspamd_srv_reply {
                        gint code;
                } spair;
                struct {
-                       gint unused;
+                       gint forced;
                } hs_loaded;
        } reply;
 };

diff --git a/src/worker.c b/src/worker.c
index 9c5f0bd40eaa1de80b844ee6df888e42485b7c8d..7598c687f286bef5d7e8ead829966ae57fb5ba78 100644 (file)
--- a/src/worker.c
+++ b/src/worker.c
@@ -293,13 +293,19 @@ rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main,
                gpointer ud)
 {
        struct rspamd_control_reply rep;
+       struct rspamd_re_cache *cache = worker->srv->cfg->re_cache;
 
-       msg_info ("loading hyperscan expressions after receiving compilation notice");
        memset (&rep, 0, sizeof (rep));
        rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED;
 
-       rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan (
-                       worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir);
+       if (!rspamd_re_cache_is_hs_loaded (cache) || cmd->cmd.hs_loaded.forced) {
+               msg_info ("loading hyperscan expressions after receiving compilation "
+                               "notice: %s",
+                               (!rspamd_re_cache_is_hs_loaded (cache)) ?
+                                               "new db" : "forced update");
+               rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan (
+                               worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir);
+       }
 
        if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) {
                msg_err ("cannot write reply to the control socket: %s",