diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2015-05-21 10:23:41 +0100 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2015-05-21 10:23:41 +0100 |
| commit | c44ddedf8c4950cc679073bb809e8d27b0186951 (patch) | |
| tree | 9769ffe64b6e6f6bb179d5a111047c890eac9e58 /src/libstat/tokenizers | |
| parent | 495898bcb6e2dbf7cc27762124d6340eb65fc238 (diff) | |
| download | rspamd-c44ddedf8c4950cc679073bb809e8d27b0186951.tar.gz rspamd-c44ddedf8c4950cc679073bb809e8d27b0186951.zip | |
More fixes to tokenization.
Diffstat (limited to 'src/libstat/tokenizers')
| -rw-r--r-- | src/libstat/tokenizers/tokenizers.c | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c index 086b1719b..586b3079b 100644 --- a/src/libstat/tokenizers/tokenizers.c +++ b/src/libstat/tokenizers/tokenizers.c @@ -223,6 +223,7 @@ rspamd_tokenizer_get_word (rspamd_fstring_t * buf, if (ex != NULL && p - buf->begin == (gint)ex->pos) { token->begin = "exception"; token->len = sizeof ("exception") - 1; + processed = token->len; state = skip_exception; continue; } @@ -257,9 +258,11 @@ set_token: *rl = processed; } - token->len = p - token->begin; - g_assert (token->len > 0); - *cur = p; + if (token->len == 0) { + token->len = p - token->begin; + g_assert (token->len > 0); + *cur = p; + } return TRUE; } @@ -275,7 +278,7 @@ rspamd_tokenize_text (gchar *text, gsize len, gboolean is_utf, GList *cur = exceptions; token_get_function func; - if (len == 0 || text == NULL) { + if (text == NULL) { return NULL; } |