diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-08 09:41:19 +0000 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-08 09:41:57 +0000 |
| commit | 9ebb11d00a678e97f14b4cd33a68c3e5cd385392 (patch) | |
| tree | 2a701df0af31d895766325f8d5d759eec7a50b47 /src/libstat/stat_process.c | |
| parent | ad33efe0d2a35eee8a3a47a6fdffcae6d6b8e185 (diff) | |
| download | rspamd-9ebb11d00a678e97f14b4cd33a68c3e5cd385392.tar.gz rspamd-9ebb11d00a678e97f14b4cd33a68c3e5cd385392.zip | |
[Minor] Move subject tokenisation to a separate routine
Issue: #2623
Diffstat (limited to 'src/libstat/stat_process.c')
| -rw-r--r-- | src/libstat/stat_process.c | 27 |
1 files changed, 1 insertions, 26 deletions
diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index 2641226d5..228b6b87b 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -313,10 +313,8 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, { struct rspamd_mime_text_part *part; rspamd_cryptobox_hash_state_t hst; - rspamd_stat_token_t *tok; rspamd_token_t *st_tok; GArray *words; - gchar *sub = NULL; guint i, reserved_len = 0; gdouble *pdiff; guchar hout[rspamd_cryptobox_HASHBYTES]; @@ -361,29 +359,8 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, } if (task->subject != NULL) { - sub = task->subject; - } - - if (sub != NULL) { - UText utxt = UTEXT_INITIALIZER; - UErrorCode uc_err = U_ZERO_ERROR; - gsize slen = strlen (sub); - - utext_openUTF8 (&utxt, - sub, - slen, - &uc_err); - - words = rspamd_tokenize_text (sub, slen, &utxt, RSPAMD_TOKENIZE_UTF, - NULL, NULL, NULL); - + words = rspamd_tokenize_subject (task); if (words != NULL) { - - for (i = 0; i < words->len; i ++) { - tok = &g_array_index (words, rspamd_stat_token_t, i); - tok->flags |= RSPAMD_STAT_TOKEN_FLAG_SUBJECT; - } - st_ctx->tokenizer->tokenize_func (st_ctx, task->task_pool, words, @@ -394,8 +371,6 @@ rspamd_stat_process_tokenize (struct rspamd_stat_ctx *st_ctx, rspamd_mempool_add_destructor (task->task_pool, rspamd_array_free_hard, words); } - - utext_close (&utxt); } rspamd_stat_tokenize_parts_metadata (st_ctx, task); |