From: Vsevolod Stakhov <vsevolod@rambler-co.ru>
Date: Fri, 11 Feb 2011 13:01:23 +0000 (+0300)
Subject: * Tokenize subject using osb tokenizer.
X-Git-Tag: 0.3.7~49
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=0aab87dab86cc9186e267b1448373411d816136e;p=rspamd.git

* Tokenize subject using osb tokenizer.
---

diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index 1c1f6d9a8..9c3e7f1f9 100644
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -254,38 +254,30 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
 void
 tokenize_subject (struct worker_task *task, GTree ** tree)
 {
-	f_str_t                         subject, subject_name;
+	f_str_t                         subject;
 	const gchar                    *sub;
 	token_node_t                   *new = NULL;
+	struct tokenizer               *osb_tokenizer;
 
 	if (*tree == NULL) {
 		*tree = g_tree_new (token_node_compare_func);
 		memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_tree_destroy, *tree);
 	}
 
-	subject_name.begin = "Subject:";
-	subject_name.len = sizeof ("Subject:") - 1;
+	osb_tokenizer = get_tokenizer ("osb");
 
 	/* Try to use pre-defined subject */
 	if (task->subject != NULL) {
 		new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
 		subject.begin = task->subject;
 		subject.len = strlen (task->subject);
-		new->h1 = fstrhash (&subject_name) * primes[0];
-		new->h2 = fstrhash (&subject) * primes[1];
-		if (g_tree_lookup (*tree, new) == NULL) {
-			g_tree_insert (*tree, new, new);
-		}
+		osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree);
 	}
 	if ((sub = g_mime_message_get_subject (task->message)) != NULL) {
 		new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
 		subject.begin = (gchar *)sub;
 		subject.len = strlen (sub);
-		new->h1 = fstrhash (&subject_name) * primes[0];
-		new->h2 = fstrhash (&subject) * primes[1];
-		if (g_tree_lookup (*tree, new) == NULL) {
-			g_tree_insert (*tree, new, new);
-		}
+		osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree);
 	}
 }