diff options
Diffstat (limited to 'src/libstat/tokenizers/tokenizers.c')
| -rw-r--r-- | src/libstat/tokenizers/tokenizers.c | 7 |
1 files changed, 4 insertions, 3 deletions
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 0902ceb05..ac7f8be85 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -393,7 +393,7 @@ start_over: if (rspamd_utf_word_valid (text, text + len, last, p)) { token.begin = text + last; token.len = p - last; - token.flags = 0; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; } } } @@ -407,7 +407,7 @@ start_over: if (rspamd_utf_word_valid (text, text + len, last, p)) { token.begin = text + last; token.len = p - last; - token.flags = 0; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; } } else { @@ -415,7 +415,7 @@ start_over: if (rspamd_utf_word_valid (text, text + len, last, p)) { token.begin = text + last; token.len = p - last; - token.flags = 0; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; } } } @@ -423,6 +423,7 @@ start_over: if (rspamd_utf_word_valid (text, text + len, last, p)) { token.begin = text + last; token.len = p - last; + token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT; } } |