[Minor] Allow to tokenize task externally in API

diff --git a/src/libstat/stat_api.h b/src/libstat/stat_api.h
index a4e02a591c767410a4963572ab645f3a785217be..1ade4ccd56d06f4ff4cc4d9e02041399ad53b11f 100644 (file)
--- a/src/libstat/stat_api.h
+++ b/src/libstat/stat_api.h
@@ -39,6 +39,8 @@ typedef struct rspamd_stat_token_s {
        guint flags;
 } rspamd_stat_token_t;
 
+struct rspamd_stat_ctx;
+
 /**
  * The results of statistics processing:
  * - error
@@ -62,6 +64,14 @@ void rspamd_stat_init (struct rspamd_config *cfg, struct event_base *ev_base);
  */
 void rspamd_stat_close (void);
 
+/**
+ * Tokenize task
+ * @param st_ctx
+ * @param task
+ */
+void rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
+                                                         struct rspamd_task *task);
+
 /**
  * Classify the task specified and insert symbols if needed
  * @param task

diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c
index 4f7f4e7033642e14da05ef79c4983c496a1639d2..eb8b8bd7c6e25268ae759f0647a444b2bd8ca984 100644 (file)
--- a/src/libstat/stat_process.c
+++ b/src/libstat/stat_process.c
@@ -307,7 +307,7 @@ rspamd_stat_tokenize_parts_metadata (struct rspamd_stat_ctx *st_ctx,
 /*
  * Tokenize task using the tokenizer specified
  */
-static void
+void
 rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
                struct rspamd_task *task)
 {
@@ -322,6 +322,12 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx,
        guchar hout[rspamd_cryptobox_HASHBYTES];
        gchar *b32_hout;
 
+       if (st_ctx == NULL) {
+               st_ctx = rspamd_stat_get_ctx ();
+       }
+
+       g_assert (st_ctx != NULL);
+
        for (i = 0; i < task->text_parts->len; i++) {
                part = g_ptr_array_index (task->text_parts, i);
 
@@ -409,7 +415,10 @@ rspamd_stat_preprocess (struct rspamd_stat_ctx *st_ctx,
        struct rspamd_statfile *st;
        gpointer bk_run;
 
-       rspamd_stat_process_tokenize (st_ctx, task);
+       if (task->tokens == NULL) {
+               rspamd_stat_process_tokenize (st_ctx, task);
+       }
+
        task->stat_runtimes = g_ptr_array_sized_new (st_ctx->statfiles->len);
        g_ptr_array_set_size (task->stat_runtimes, st_ctx->statfiles->len);
        rspamd_mempool_add_destructor (task->task_pool,