From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Wed, 20 May 2015 21:37:33 +0000 (+0100)
Subject: Add a simple unit test for tokenizer.
X-Git-Tag: 0.9.4~12
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=8b724ff651736b296f96e35730989ac2710c66e1;p=rspamd.git

Add a simple unit test for tokenizer.
---

diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua
new file mode 100644
index 000000000..de15a6bd7
--- /dev/null
+++ b/test/lua/unit/tokenizer.lua
@@ -0,0 +1,23 @@
+context("Text tokenization test", function()
+  local util = require "rspamd_util"
+  local logger = require "rspamd_logger"
+  test("Tokenize simple text", function()
+    local cases = {
+      {"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer mattis, nibh",
+        {"Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit",
+        "Integer", "mattis", "nibh"
+        }
+      },
+    }
+    
+    for _,c in ipairs(cases) do
+      local w = util.tokenize_text(c[1])
+      assert_not_nil(w, "cannot tokenize " .. c[1])
+      
+      for i,wrd in ipairs(w) do
+        logger.infox('%1:%2', i, wrd)
+        assert_equal(wrd, c[2][i])
+      end
+    end
+  end)
+end)
\ No newline at end of file