[Fix] Fix another tokenization issue

MFH: rspamd-1.6

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 74c4f5460f9dd845fa40f85b5e745bb50140d873..c4f814fce8d7a9b3fe5e571206165ad67df3a035 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -269,15 +269,45 @@ set_token:
        return TRUE;
 
 process_exception:
+       if (token->len == 0 && processed > 0) {
+               token->len = p - token->begin;
+               g_assert (token->len > 0);
+       }
+
        if (ex->type == RSPAMD_EXCEPTION_URL) {
                token->begin = "!!EX!!";
                token->len = sizeof ("!!EX!!") - 1;
                token->flags = RSPAMD_STAT_TOKEN_FLAG_EXCEPTION;
                processed = token->len;
        }
-       *cur = p + ex->len;
+
+       p += ex->len;
+
+       /* We need to skip all exceptions that are within this exception */
        *exceptions = g_list_next (*exceptions);
 
+       while (*exceptions) {
+               ex = (*exceptions)->data;
+
+               if (ex->pos < p - buf->begin) {
+                       /* Nested exception */
+                       if (ex->pos + ex->len > p - buf->begin) {
+                               /*
+                                * We have somehow overlapping nesting exception,
+                                * extend current offset
+                                */
+                               p = buf->begin + ex->pos + ex->len;
+                       }
+
+                       *exceptions = g_list_next (*exceptions);
+               }
+               else {
+                       break;
+               }
+       }
+
+       *cur = p;
+
        if (rl) {
                *rl = processed;
        }