Fix tokenization of the last token in a message.

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index eebc57c22db8e39bf47f01947424f533d67cb8df..4b8f0cfb392d7f25f5989846f011536286b069a1 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -295,7 +295,7 @@ rspamd_tokenize_text (gchar *text, gsize len, gboolean is_utf,
        res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_fstring_t), 128);
 
        while (func (&buf, &pos, &token, &cur, is_utf, &l)) {
-               if (min_len > 0 && l < min_len) {
+               if (l == 0 || (min_len > 0 && l < min_len)) {
                        token.begin = pos;
                        continue;
                }