diff options
| author | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2011-06-03 20:23:13 +0400 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2011-06-03 20:23:13 +0400 |
| commit | 92de380c2c5e8ce7073ce979df4e5c7868e52bb6 (patch) | |
| tree | 27be3202d27f129f3d94d90298a4d1e0ecf2c281 /src/tokenizers/tokenizers.c | |
| parent | 83a9452974ec2f9c7be262a77e54a1ea2557c795 (diff) | |
| download | rspamd-92de380c2c5e8ce7073ce979df4e5c7868e52bb6.tar.gz rspamd-92de380c2c5e8ce7073ce979df4e5c7868e52bb6.zip | |
* Skip short utf words in statistics
Diffstat (limited to 'src/tokenizers/tokenizers.c')
| -rw-r--r-- | src/tokenizers/tokenizers.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c index 5af3fe6d5..9e41a9101 100644 --- a/src/tokenizers/tokenizers.c +++ b/src/tokenizers/tokenizers.c @@ -239,13 +239,13 @@ tokenize_subject (struct worker_task *task, GTree ** tree) new = memory_pool_alloc (task->task_pool, sizeof (token_node_t)); subject.begin = task->subject; subject.len = strlen (task->subject); - osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE); + osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE, TRUE); } if ((sub = g_mime_message_get_subject (task->message)) != NULL) { new = memory_pool_alloc (task->task_pool, sizeof (token_node_t)); subject.begin = (gchar *)sub; subject.len = strlen (sub); - osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE); + osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE, TRUE); } } |