From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Wed, 16 Jun 2021 15:40:53 +0000 (+0100)
Subject: [Fix] Fix tokenization near exceptions
X-Git-Tag: 3.0~288
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=ce90a17cc548782f9fa204148750f2c5c192e8d7;p=rspamd.git

[Fix] Fix tokenization near exceptions
---

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 8d6d93add..2dd4a6f5a 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -459,8 +459,8 @@ start_over:
 									ex->pos)) {
 								token.original.begin = text + last;
 								token.original.len = ex->pos - last;
-								token.flags = 0;
-								g_array_append_val (res, token);
+								token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+											  RSPAMD_STAT_TOKEN_FLAG_UTF;
 							}
 
 							/* Process the current exception */