Rework hyperscan notifies to reduce noise level

6 files changed, 62 insertions, 23 deletions

diff --git a/src/hs_helper.c b/src/hs_helper.c
index bf317cbbb..771d74d7a 100644
--- a/src/hs_helper.c
+++ b/src/hs_helper.c
@@ -176,7 +176,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
 	}
 
 	if ((ncompiled = rspamd_re_cache_compile_hyperscan (ctx->cfg->re_cache,
-			ctx->hs_dir, ctx->max_time,
+			ctx->hs_dir, ctx->max_time, !forced,
 			&err)) == -1) {
 		msg_err ("failed to compile re cache: %e", err);
 		g_error_free (err);
@@ -184,8 +184,11 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
 		return FALSE;
 	}
 
-	msg_info ("compiled %d regular expressions to the hyperscan tree",
-			ncompiled);
+	if (ncompiled > 0) {
+		msg_info ("compiled %d regular expressions to the hyperscan tree",
+				ncompiled);
+		forced = TRUE;
+	}
 
 	/*
 	 * Do not send notification unless all other workers are started
@@ -198,6 +201,7 @@ rspamd_rs_compile (struct hs_helper_ctx *ctx, struct rspamd_worker *worker,
 
 	srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED;
 	srv_cmd.cmd.hs_loaded.cache_dir = ctx->hs_dir;
+	srv_cmd.cmd.hs_loaded.forced = forced;
 
 	rspamd_srv_send_command (worker, ctx->ev_base, &srv_cmd, NULL, NULL);
 
diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c
index 4434576c8..e0de03184 100644
--- a/src/libserver/re_cache.c
+++ b/src/libserver/re_cache.c
@@ -100,6 +100,7 @@ struct rspamd_re_cache {
 	guint max_re_data;
 	gchar hash[rspamd_cryptobox_HASHBYTES + 1];
 #ifdef WITH_HYPERSCAN
+	gboolean hyperscan_loaded;
 	gboolean disable_hyperscan;
 	hs_platform_info_t plt;
 #endif
@@ -186,11 +187,26 @@ rspamd_re_cache_new (void)
 	cache->re_classes = g_hash_table_new (g_int64_hash, g_int64_equal);
 	cache->nre = 0;
 	cache->re = g_ptr_array_new_full (256, rspamd_re_cache_elt_dtor);
+#ifdef WITH_HYPERSCAN
+	cache->hyperscan_loaded = FALSE;
+#endif
 	REF_INIT_RETAIN (cache, rspamd_re_cache_destroy);
 
 	return cache;
 }
 
+gboolean
+rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache)
+{
+	g_assert (cache != NULL);
+
+#ifdef WITH_HYPERSCAN
+	return cache->hyperscan_loaded;
+#else
+	return FALSE;
+#endif
+}
+
 rspamd_regexp_t *
 rspamd_re_cache_add (struct rspamd_re_cache *cache, rspamd_regexp_t *re,
 		enum rspamd_re_type type, gpointer type_data, gsize datalen)
@@ -1011,7 +1027,7 @@ rspamd_re_cache_is_finite (struct rspamd_re_cache *cache,
 
 gint
 rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
-		const char *cache_dir, gdouble max_time,
+		const char *cache_dir, gdouble max_time, gboolean silent,
 		GError **err)
 {
 	g_assert (cache != NULL);
@@ -1051,24 +1067,27 @@ rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
 			g_assert (fd != -1);
 			lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET);
 			read (fd, &n, sizeof (n));
-			total += n;
 			close (fd);
 
 			if (re_class->type_len > 0) {
-				msg_info_re_cache (
-						"skip already valid class %s(%*s) to cache %6s, %d regexps",
-						rspamd_re_cache_type_to_string (re_class->type),
-						(gint) re_class->type_len - 1,
-						re_class->type_data,
-						re_class->hash,
-						n);
+				if (!silent) {
+					msg_info_re_cache (
+							"skip already valid class %s(%*s) to cache %6s, %d regexps",
+							rspamd_re_cache_type_to_string (re_class->type),
+							(gint) re_class->type_len - 1,
+							re_class->type_data,
+							re_class->hash,
+							n);
+				}
 			}
 			else {
-				msg_info_re_cache (
-						"skip already valid class %s to cache %6s, %d regexps",
-						rspamd_re_cache_type_to_string (re_class->type),
-						re_class->hash,
-						n);
+				if (!silent) {
+					msg_info_re_cache (
+							"skip already valid class %s to cache %6s, %d regexps",
+							rspamd_re_cache_type_to_string (re_class->type),
+							re_class->hash,
+							n);
+				}
 			}
 
 			continue;
@@ -1491,6 +1510,7 @@ rspamd_re_cache_load_hyperscan (struct rspamd_re_cache *cache,
 	}
 
 	msg_info_re_cache ("hyperscan database of %d regexps has been loaded", total);
+	cache->hyperscan_loaded = TRUE;
 
 	return TRUE;
 #endif
diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h
index ae5cf9019..8358f146e 100644
--- a/src/libserver/re_cache.h
+++ b/src/libserver/re_cache.h
@@ -84,6 +84,13 @@ void rspamd_re_cache_init (struct rspamd_re_cache *cache,
 		struct rspamd_config *cfg);
 
 /**
+ * Returns true when hyperscan is loaded
+ * @param cache
+ * @return
+ */
+gboolean rspamd_re_cache_is_hs_loaded (struct rspamd_re_cache *cache);
+
+/**
  * Get runtime data for a cache
  */
 struct rspamd_re_runtime* rspamd_re_cache_runtime_new (struct rspamd_re_cache *cache);
@@ -145,7 +152,7 @@ enum rspamd_re_type rspamd_re_cache_type_from_string (const char *str);
  * Compile expressions to the hyperscan tree and store in the `cache_dir`
  */
 gint rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache,
-		const char *cache_dir, gdouble max_time,
+		const char *cache_dir, gdouble max_time, gboolean silent,
 		GError **err);
 
 
diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
index 6696cd183..a91431cac 100644
--- a/src/libserver/rspamd_control.c
+++ b/src/libserver/rspamd_control.c
@@ -716,9 +716,9 @@ rspamd_srv_handler (gint fd, short what, gpointer ud)
 				 * workers
 				 */
 				wcmd.cmd.hs_loaded.cache_dir = cmd.cmd.hs_loaded.cache_dir;
+				wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced;
 				rspamd_control_broadcast_cmd (srv, &wcmd,
 						rspamd_control_hs_io_handler, NULL);
-				rdata->rep.reply.hs_loaded.unused = 0;
 				break;
 			default:
 				msg_err ("unknown command type: %d", cmd.type);
diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h
index a15ace0d6..f8113cbb4 100644
--- a/src/libserver/rspamd_control.h
+++ b/src/libserver/rspamd_control.h
@@ -64,6 +64,7 @@ struct rspamd_control_command {
 		} recompile;
 		struct {
 			gpointer cache_dir;
+			gboolean forced;
 		} hs_loaded;
 		struct {
 			guint unused;
@@ -112,6 +113,7 @@ struct rspamd_srv_command {
 		} spair;
 		struct {
 			gpointer cache_dir;
+			gboolean forced;
 		} hs_loaded;
 	} cmd;
 };
@@ -124,7 +126,7 @@ struct rspamd_srv_reply {
 			gint code;
 		} spair;
 		struct {
-			gint unused;
+			gint forced;
 		} hs_loaded;
 	} reply;
 };
diff --git a/src/worker.c b/src/worker.c
index 9c5f0bd40..7598c687f 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -293,13 +293,19 @@ rspamd_worker_hyperscan_ready (struct rspamd_main *rspamd_main,
 		gpointer ud)
 {
 	struct rspamd_control_reply rep;
+	struct rspamd_re_cache *cache = worker->srv->cfg->re_cache;
 
-	msg_info ("loading hyperscan expressions after receiving compilation notice");
 	memset (&rep, 0, sizeof (rep));
 	rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED;
 
-	rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan (
-			worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir);
+	if (!rspamd_re_cache_is_hs_loaded (cache) || cmd->cmd.hs_loaded.forced) {
+		msg_info ("loading hyperscan expressions after receiving compilation "
+				"notice: %s",
+				(!rspamd_re_cache_is_hs_loaded (cache)) ?
+						"new db" : "forced update");
+		rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan (
+				worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir);
+	}
 
 	if (write (fd, &rep, sizeof (rep)) != sizeof (rep)) {
 		msg_err ("cannot write reply to the control socket: %s",