From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 19 Jun 2018 16:01:46 +0000 (+0100)
Subject: [Minor] Allow to tokenize task externally in API
X-Git-Tag: 1.7.7~81
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=5bf6c2c3ab9d103705e41d0a47e96a47946d8aa0;p=rspamd.git

[Minor] Allow to tokenize task externally in API
---

diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h
index a4e02a591..1ade4ccd5 100644
--- a/src/libstat/stat_api.h
+++ b/src/libstat/stat_api.h
@@ -39,6 +39,8 @@ typedef struct rspamd_stat_token_s {
 	guint flags;
 } rspamd_stat_token_t;
 
+struct rspamd_stat_ctx;
+
 /**
  * The results of statistics processing:
  * - error
@@ -62,6 +64,14 @@ void rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base);
  */
 void rspamd_stat_close (void);
 
+/**
+ * Tokenize task
+ * @param st_ctx
+ * @param task
+ */
+void rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
+							  struct rspamd_task *task);
+
 /**
  * Classify the task specified and insert symbols if needed
  * @param task
diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c
index 4f7f4e703..eb8b8bd7c 100644
--- a/src/libstat/stat_process.c
+++ b/src/libstat/stat_process.c
@@ -307,7 +307,7 @@ rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx,
 /*
  * Tokenize task using the tokenizer specified
  */
-static void
+void
 rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
 		struct rspamd_task *task)
 {
@@ -322,6 +322,12 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
 	guchar hout[rspamd_cryptobox_HASHBYTES];
 	gchar *b32_hout;
 
+	if (st_ctx == NULL) {
+		st_ctx = rspamd_stat_get_ctx ();
+	}
+
+	g_assert (st_ctx != NULL);
+
 	for (i = 0; i < task->text_parts->len; i++) {
 		part = g_ptr_array_index (task->text_parts, i);
 
@@ -409,7 +415,10 @@ rspamd_stat_preprocess (struct rspamd_stat_ctx *st_ctx,
 	struct rspamd_statfile *st;
 	gpointer bk_run;
 
-	rspamd_stat_process_tokenize (st_ctx, task);
+	if (task->tokens == NULL) {
+		rspamd_stat_process_tokenize (st_ctx, task);
+	}
+
 	task->stat_runtimes = g_ptr_array_sized_new (st_ctx->statfiles->len);
 	g_ptr_array_set_size (task->stat_runtimes, st_ctx->statfiles->len);
 	rspamd_mempool_add_destructor (task->task_pool,