Use not common name for tokenization exceptions.

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 586b3079b0c6739196cbc27e4f8aa0aaa25efd9f..51ef9038dc80cb783034a6e541b8bd11202e1834 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -221,8 +221,8 @@ rspamd_tokenizer_get_word (rspamd_fstring_t * buf,
                switch (state) {
                case skip_delimiters:
                        if (ex != NULL && p - buf->begin == (gint)ex->pos) {
-                               token->begin = "exception";
-                               token->len = sizeof ("exception") - 1;
+                               token->begin = "!!EX!!";
+                               token->len = sizeof ("!!EX!!") - 1;
                                processed = token->len;
                                state = skip_exception;
                                continue;

diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua
index 6aefeec6ff6e4a297a0b895c33c8df1f4cbf6c72..e05f74d86bf907d56fe3658595e4485051bf10b8 100644 (file)
--- a/test/lua/unit/tokenizer.lua
+++ b/test/lua/unit/tokenizer.lua
@@ -67,31 +67,31 @@ context("Text tokenization test", function()
     local cases = {
       {"word https://example.com/path word",
         {{5, 24}},
-        {"word", "exception", "word"}
+        {"word", "!!EX!!", "word"}
       },
       {"համար https://example.com/path համար",
         {{11, 24}},
-        {"համար", "exception", "համար"}
+        {"համար", "!!EX!!", "համար"}
       },
       {"word https://example.com/path https://example.com/path word",
         {{5, 24}, {30, 24}},
-        {"word", "exception", "exception", "word"}
+        {"word", "!!EX!!", "!!EX!!", "word"}
       },
       {"word https://example.com/path https://example.com/path",
         {{5, 24}, {30, 24}},
-        {"word", "exception", "exception"}
+        {"word", "!!EX!!", "!!EX!!"}
       },
       {"https://example.com/path https://example.com/path word",
         {{0, 24}, {25, 24}},
-        {"exception", "exception", "word"}
+        {"!!EX!!", "!!EX!!", "word"}
       },
       {"https://example.com/path https://example.com/path",
         {{0, 24}, {25, 24}},
-        {"exception", "exception"}
+        {"!!EX!!", "!!EX!!"}
       },
       {",,,,https://example.com/path https://example.com/path    ",
         {{4, 24}, {29, 24}},
-        {"exception", "exception"}
+        {"!!EX!!", "!!EX!!"}
       },
     }