From ce90a17cc548782f9fa204148750f2c5c192e8d7 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 16 Jun 2021 16:40:53 +0100 Subject: [PATCH] [Fix] Fix tokenization near exceptions --- src/libstat/tokenizers/tokenizers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 8d6d93add..2dd4a6f5a 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -459,8 +459,8 @@ start_over: ex->pos)) { token.original.begin = text + last; token.original.len = ex->pos - last; - token.flags = 0; - g_array_append_val (res, token); + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } /* Process the current exception */ -- 2.39.5