Fix incorrect calculating of token length.

diff --git a/src/tokenizers/osb.c b/src/tokenizers/osb.c
index bc57255cb50a70e59e493204777ab9bc589ec57b..8c1b4618ab96dbfca6821f2f6b75f80be6bb7931 100644 (file)
--- a/src/tokenizers/osb.c
+++ b/src/tokenizers/osb.c
@@ -86,6 +86,7 @@ osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t * pool, f_str_t *
                }
                k ++;
                token.begin = res;
+               g_assert (k < 10000);
        }
 
        return TRUE;

diff --git a/src/tokenizers/tokenizers.c b/src/tokenizers/tokenizers.c
index be73e506d4e7fed220f68eda25c908d104a2b466..16dc763ed03240e3c6ac62bbd9b140005f556835 100644 (file)
--- a/src/tokenizers/tokenizers.c
+++ b/src/tokenizers/tokenizers.c
@@ -148,7 +148,7 @@ get_next_word (f_str_t * buf, f_str_t * token, GList **exceptions)
                if (ex != NULL && ex->pos == pos) {
                        /* Go to the next exception */
                        *exceptions = g_list_next (*exceptions);
-                       return p + ex->len + 1;
+                       return p + ex->len;
                }
                pos++;
                p++;
@@ -160,7 +160,7 @@ get_next_word (f_str_t * buf, f_str_t * token, GList **exceptions)
        while (remain > 0 && !t_delimiters[*p]) {
                if (ex != NULL && ex->pos == pos) {
                        *exceptions = g_list_next (*exceptions);
-                       return p + ex->len + 1;
+                       return p + ex->len;
                }
                token->len++;
                pos++;