aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizers
diff options
context:
space:
mode:
authorVsevolod Stakhov <vsevolod@rambler-co.ru>2011-02-11 16:01:23 +0300
committerVsevolod Stakhov <vsevolod@rambler-co.ru>2011-02-11 16:01:23 +0300
commit0aab87dab86cc9186e267b1448373411d816136e (patch)
treeea679c94828cad721ec9f323d66c807eba3afd62 /src/tokenizers
parent8f97c6a684a47ff25166c04a1b0e96e20d4b2302 (diff)
downloadrspamd-0aab87dab86cc9186e267b1448373411d816136e.tar.gz
rspamd-0aab87dab86cc9186e267b1448373411d816136e.zip
* Tokenize subject using osb tokenizer.
Diffstat (limited to 'src/tokenizers')
-rw-r--r--src/tokenizers/tokenizers.c18
1 files changed, 5 insertions, 13 deletions
diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index 1c1f6d9a8..9c3e7f1f9 100644
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -254,38 +254,30 @@ tokenize_headers (memory_pool_t * pool, struct worker_task *task, GTree ** tree)
void
tokenize_subject (struct worker_task *task, GTree ** tree)
{
- f_str_t subject, subject_name;
+ f_str_t subject;
const gchar *sub;
token_node_t *new = NULL;
+ struct tokenizer *osb_tokenizer;
if (*tree == NULL) {
*tree = g_tree_new (token_node_compare_func);
memory_pool_add_destructor (task->task_pool, (pool_destruct_func) g_tree_destroy, *tree);
}
- subject_name.begin = "Subject:";
- subject_name.len = sizeof ("Subject:") - 1;
+ osb_tokenizer = get_tokenizer ("osb");
/* Try to use pre-defined subject */
if (task->subject != NULL) {
new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
subject.begin = task->subject;
subject.len = strlen (task->subject);
- new->h1 = fstrhash (&subject_name) * primes[0];
- new->h2 = fstrhash (&subject) * primes[1];
- if (g_tree_lookup (*tree, new) == NULL) {
- g_tree_insert (*tree, new, new);
- }
+ osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree);
}
if ((sub = g_mime_message_get_subject (task->message)) != NULL) {
new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
subject.begin = (gchar *)sub;
subject.len = strlen (sub);
- new->h1 = fstrhash (&subject_name) * primes[0];
- new->h2 = fstrhash (&subject) * primes[1];
- if (g_tree_lookup (*tree, new) == NULL) {
- g_tree_insert (*tree, new, new);
- }
+ osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree);
}
}