diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-30 15:03:00 +0000 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2018-11-30 15:03:00 +0000 |
| commit | 8857f667412c2db45c5d346575db3eb1cf398b04 (patch) | |
| tree | 46bcc4157c54ee8811f6618760d1f0ae6c8b6624 /src/libmime | |
| parent | 7fde20c6c535c8bee0ba418fb48cb93b0fe089bb (diff) | |
| download | rspamd-8857f667412c2db45c5d346575db3eb1cf398b04.tar.gz rspamd-8857f667412c2db45c5d346575db3eb1cf398b04.zip | |
[Minor] Count words based on text words
Diffstat (limited to 'src/libmime')
| -rw-r--r-- | src/libmime/lang_detection.c | 6 | ||||
| -rw-r--r-- | src/libmime/message.c | 8 | ||||
| -rw-r--r-- | src/libmime/message.h | 1 |
3 files changed, 10 insertions, 5 deletions
diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index f5a175ad8..82e5fc2ff 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -1665,11 +1665,11 @@ rspamd_language_detector_detect (struct rspamd_task *task, } if (!ret) { - if (part->utf_words->len < default_short_text_limit) { + if (part->nwords < default_short_text_limit) { r = rs_detect_none; msg_debug_lang_det ("text is too short for trigramms detection: " "%d words; at least %d words required", - (int)part->utf_words->len, + (int)part->nwords, (int)default_short_text_limit); rspamd_language_detector_set_language (task, part, "en"); candidates = kh_init (rspamd_candidates_hash); @@ -1728,7 +1728,7 @@ rspamd_language_detector_detect (struct rspamd_task *task, cbd.std = std; cbd.flags = RSPAMD_LANG_FLAG_DEFAULT; - if (part->utf_words->len < default_words / 2) { + if (part->nwords < default_words / 2) { cbd.flags |= RSPAMD_LANG_FLAG_SHORT; } } diff --git a/src/libmime/message.c b/src/libmime/message.c index 3c29b1170..411b3bf03 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -89,6 +89,10 @@ rspamd_mime_part_extract_words (struct rspamd_task *task, } } + if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) { + part->nwords ++; + } + if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE| RSPAMD_STAT_TOKEN_FLAG_NORMALISED| RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES)) { @@ -96,7 +100,7 @@ rspamd_mime_part_extract_words (struct rspamd_task *task, } } - if (part->utf_words && part->utf_words->len) { + if (part->utf_words->len) { gdouble *avg_len_p, *short_len_p; avg_len_p = rspamd_mempool_get_variable (task->task_pool, @@ -1294,7 +1298,7 @@ rspamd_message_process (struct rspamd_task *task) rspamd_mime_part_extract_words (task, text_part); if (text_part->utf_words) { - total_words += text_part->utf_words->len; + total_words += text_part->nwords; } } diff --git a/src/libmime/message.h b/src/libmime/message.h index ed9dfef6e..29f777c3b 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -112,6 +112,7 @@ struct rspamd_mime_text_part { guint flags; guint nlines; guint spaces; + guint nwords; guint non_ascii_chars; guint ascii_chars; guint double_spaces; |