Make words from text parts.

diff --git a/src/libmime/message.c b/src/libmime/message.c
index 0e33967caf744472308c400e2b69eaf219ce7d46..f465b1c06f3832703ebc629e069c8b642ace14cd 100644 (file)
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -30,6 +30,8 @@
 #include "html.h"
 #include "images.h"
 #include "utlist.h"
+#include "tokenizers/tokenizers.h"
+
 #include <iconv.h>
 
 #define RECURSION_LIMIT 30
@@ -1033,6 +1035,8 @@ process_text_part (struct rspamd_task *task,
 {
        struct mime_text_part *text_part;
        const gchar *cd;
+       gchar *pos;
+       rspamd_fstring_t token, buf;
 
        /* Skip attachements */
 #ifndef GMIME24
@@ -1056,7 +1060,6 @@ process_text_part (struct rspamd_task *task,
 
        if (g_mime_content_type_is_type (type, "text",
                "html") || g_mime_content_type_is_type (type, "text", "xhtml")) {
-               debug_task ("got urls from text/html part");
 
                text_part =
                        rspamd_mempool_alloc0 (task->task_pool,
@@ -1097,7 +1100,6 @@ process_text_part (struct rspamd_task *task,
                task->text_parts = g_list_prepend (task->text_parts, text_part);
        }
        else if (g_mime_content_type_is_type (type, "text", "*")) {
-               debug_task ("got urls from text/plain part");
 
                text_part =
                        rspamd_mempool_alloc0 (task->task_pool,
@@ -1120,6 +1122,20 @@ process_text_part (struct rspamd_task *task,
                rspamd_fuzzy_from_text_part (text_part, task->task_pool, task->cfg->max_diff);
                task->text_parts = g_list_prepend (task->text_parts, text_part);
        }
+       else {
+               return;
+       }
+
+       /* Post process part */
+       buf.begin = text_part->content->data;
+       buf.len = text_part->content->len;
+       buf.size = buf.len;
+
+       text_part->words = g_array_new (FALSE, FALSE, sizeof (rspamd_fstring_t));
+       while ((pos = rspamd_tokenizer_get_word (&buf,
+                       &token, &text_part->urls_offset)) != NULL) {
+               g_array_append_val (text_part->words, token);
+       }
 }
 
 #ifdef GMIME24

diff --git a/src/libmime/message.h b/src/libmime/message.h
index ddc5939fa9d264c7c2e64e31d483cbe81aa6f9eb..8287db9b022d13ee3f408ca3a5f73a4c16db7678 100644 (file)
--- a/src/libmime/message.h
+++ b/src/libmime/message.h
@@ -30,12 +30,13 @@ struct mime_text_part {
        GByteArray *orig;
        GByteArray *content;
        GNode *html_nodes;
-       GList *urls_offset;                                         /**< list of offsets of urls                                                */
+       GList *urls_offset;     /**< list of offsets of urls                                            */
        rspamd_fuzzy_t *fuzzy;
        rspamd_fuzzy_t *double_fuzzy;
        GMimeObject *parent;
        GUnicodeScript script;
        rspamd_fstring_t *diff_str;
+       GArray *words;
 };
 
 struct received_header {

diff --git a/src/libserver/task.c b/src/libserver/task.c
index b6a910a911a02427146bb2ba2ad94e71a6653e5b..bfa9184fb8ca99441026568819330e3a33cd4c0d 100644 (file)
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -226,6 +226,7 @@ rspamd_task_free (struct rspamd_task *task, gboolean is_soft)
 {
        GList *part;
        struct mime_part *p;
+       struct mime_text_part *tp;
 
        if (task) {
                debug_task ("free pointer %p", task);
@@ -236,6 +237,15 @@ rspamd_task_free (struct rspamd_task *task, gboolean is_soft)
                        g_list_free_1 (part);
                }
                if (task->text_parts) {
+                       part = task->text_parts;
+                       while (part) {
+                               tp = (struct mime_text_part *)part->data;
+                               if (tp->words) {
+                                       g_array_free (tp->words, TRUE);
+                               }
+                               part = g_list_next (part);
+                       }
+
                        g_list_free (task->text_parts);
                }
                if (task->images) {

diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index 4789d8f628cd16ee30bcce569aea298dcaf3ab90..2adc86ff9a500c9e377f1c8d7e35d8a262d8a02a 100644 (file)
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -31,7 +31,7 @@
 #include "tokenizers.h"
 
 struct tokenizer tokenizers[] = {
-       {"osb-text", osb_tokenize_text, get_next_word},
+       {"osb-text", osb_tokenize_text, rspamd_tokenizer_get_word},
 };
 
 const int primes[] = {
@@ -104,7 +104,7 @@ token_node_compare_func (gconstpointer a, gconstpointer b)
 
 /* Get next word from specified f_str_t buf */
 gchar *
-get_next_word (rspamd_fstring_t * buf, rspamd_fstring_t * token, GList **exceptions)
+rspamd_tokenizer_get_word (rspamd_fstring_t * buf, rspamd_fstring_t * token, GList **exceptions)
 {
        gsize remain, pos;
        guchar *p;

diff --git a/src/tokenizers/tokenizers.h b/src/tokenizers/tokenizers.h
index b107296149e6508e2784936ba90f8364aeea969c..efce5b3075f173115b1cabc330c1691185b54590 100644 (file)
--- a/src/tokenizers/tokenizers.h
+++ b/src/tokenizers/tokenizers.h
@@ -30,7 +30,8 @@ int token_node_compare_func (gconstpointer a, gconstpointer b);
 /* Get tokenizer structure by name or return NULL if this name is not found */
 struct tokenizer * get_tokenizer (const char *name);
 /* Get next word from specified f_str_t buf */
-gchar * get_next_word (rspamd_fstring_t *buf, rspamd_fstring_t *token, GList **exceptions);
+gchar * rspamd_tokenizer_get_word (rspamd_fstring_t *buf,
+               rspamd_fstring_t *token, GList **exceptions);
 /* OSB tokenize function */
 int osb_tokenize_text (struct tokenizer *tokenizer,
        rspamd_mempool_t *pool,