diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2021-06-16 16:40:53 +0100 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2021-06-17 11:39:22 +0100 |
| commit | ce90a17cc548782f9fa204148750f2c5c192e8d7 (patch) | |
| tree | 294a02d64333b46909e48b19882273a41e002ebc /src/libstat/tokenizers | |
| parent | 75765b55213e824f21e18537e09af01e78c86ec6 (diff) | |
| download | rspamd-ce90a17cc548782f9fa204148750f2c5c192e8d7.tar.gz rspamd-ce90a17cc548782f9fa204148750f2c5c192e8d7.zip | |
[Fix] Fix tokenization near exceptions
Diffstat (limited to 'src/libstat/tokenizers')
| -rw-r--r-- | src/libstat/tokenizers/tokenizers.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 8d6d93add..2dd4a6f5a 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -459,8 +459,8 @@ start_over: ex->pos)) { token.original.begin = text + last; token.original.len = ex->pos - last; - token.flags = 0; - g_array_append_val (res, token); + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT | + RSPAMD_STAT_TOKEN_FLAG_UTF; } /* Process the current exception */ |