[Fix] Fix tokenization near exceptions

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 8d6d93addad6848003ba59973b9ddb60208c4e82..2dd4a6f5a1061e4af0ec72027145a076a8325b9c 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -459,8 +459,8 @@ start_over:
                                                                        ex->pos)) {
                                                                token.original.begin = text + last;
                                                                token.original.len = ex->pos - last;
-                                                               token.flags = 0;
-                                                               g_array_append_val (res, token);
+                                                               token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+                                                                                         RSPAMD_STAT_TOKEN_FLAG_UTF;
                                                        }
 
                                                        /* Process the current exception */