Fix critical bug in tokenization logic.

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 4b8f0cfb392d7f25f5989846f011536286b069a1..086b1719b8d3e2d53dafd38f63dd306d7f2d3bb4 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -257,7 +257,7 @@ set_token:
                *rl = processed;
        }
 
-       token->len = p - *cur;
+       token->len = p - token->begin;
        g_assert (token->len > 0);
        *cur = p;