From: Vsevolod Stakhov Date: Wed, 20 May 2015 21:37:33 +0000 (+0100) Subject: Add a simple unit test for tokenizer. X-Git-Tag: 0.9.4~12 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=8b724ff651736b296f96e35730989ac2710c66e1;p=rspamd.git Add a simple unit test for tokenizer. --- diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua new file mode 100644 index 000000000..de15a6bd7 --- /dev/null +++ b/test/lua/unit/tokenizer.lua @@ -0,0 +1,23 @@ +context("Text tokenization test", function() + local util = require "rspamd_util" + local logger = require "rspamd_logger" + test("Tokenize simple text", function() + local cases = { + {"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer mattis, nibh", + {"Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit", + "Integer", "mattis", "nibh" + } + }, + } + + for _,c in ipairs(cases) do + local w = util.tokenize_text(c[1]) + assert_not_nil(w, "cannot tokenize " .. c[1]) + + for i,wrd in ipairs(w) do + logger.infox('%1:%2', i, wrd) + assert_equal(wrd, c[2][i]) + end + end + end) +end) \ No newline at end of file