From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Thu, 21 May 2015 11:52:54 +0000 (+0100)
Subject: Use not common name for tokenization exceptions.
X-Git-Tag: 0.9.4~5
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=1c957d5ea3d1f21877fd647544486e6696ac39bd;p=rspamd.git

Use not common name for tokenization exceptions.
---

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 586b3079b..51ef9038d 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -221,8 +221,8 @@ rspamd_tokenizer_get_word (rspamd_fstring_t * buf,
 		switch (state) {
 		case skip_delimiters:
 			if (ex != NULL && p - buf->begin == (gint)ex->pos) {
-				token->begin = "exception";
-				token->len = sizeof ("exception") - 1;
+				token->begin = "!!EX!!";
+				token->len = sizeof ("!!EX!!") - 1;
 				processed = token->len;
 				state = skip_exception;
 				continue;
diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua
index 6aefeec6f..e05f74d86 100644
--- a/test/lua/unit/tokenizer.lua
+++ b/test/lua/unit/tokenizer.lua
@@ -67,31 +67,31 @@ context("Text tokenization test", function()
     local cases = {
       {"word https://example.com/path word",
         {{5, 24}},
-        {"word", "exception", "word"}
+        {"word", "!!EX!!", "word"}
       },
       {"համար https://example.com/path համար",
         {{11, 24}},
-        {"համար", "exception", "համար"}
+        {"համար", "!!EX!!", "համար"}
       },
       {"word https://example.com/path https://example.com/path word",
         {{5, 24}, {30, 24}},
-        {"word", "exception", "exception", "word"}
+        {"word", "!!EX!!", "!!EX!!", "word"}
       },
       {"word https://example.com/path https://example.com/path",
         {{5, 24}, {30, 24}},
-        {"word", "exception", "exception"}
+        {"word", "!!EX!!", "!!EX!!"}
       },
       {"https://example.com/path https://example.com/path word",
         {{0, 24}, {25, 24}},
-        {"exception", "exception", "word"}
+        {"!!EX!!", "!!EX!!", "word"}
       },
       {"https://example.com/path https://example.com/path",
         {{0, 24}, {25, 24}},
-        {"exception", "exception"}
+        {"!!EX!!", "!!EX!!"}
       },
       {",,,,https://example.com/path https://example.com/path    ",
         {{4, 24}, {29, 24}},
-        {"exception", "exception"}
+        {"!!EX!!", "!!EX!!"}
       },
     }