Update remain on tokenization.

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 744e6707e43ab2500323d8a38ddb14476dabbedd..c9b65e343ea4839f73b7b912a87b26688998f840 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -246,6 +246,7 @@ rspamd_tokenizer_get_word (rspamd_fstring_t * buf,
                        break;
                }
 
+               remain -= next_p - p;
                p = next_p;
        }