From 8b724ff651736b296f96e35730989ac2710c66e1 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 20 May 2015 22:37:33 +0100 Subject: [PATCH] Add a simple unit test for tokenizer. --- test/lua/unit/tokenizer.lua | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/lua/unit/tokenizer.lua diff --git a/test/lua/unit/tokenizer.lua b/test/lua/unit/tokenizer.lua new file mode 100644 index 000000000..de15a6bd7 --- /dev/null +++ b/test/lua/unit/tokenizer.lua @@ -0,0 +1,23 @@ +context("Text tokenization test", function() + local util = require "rspamd_util" + local logger = require "rspamd_logger" + test("Tokenize simple text", function() + local cases = { + {"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer mattis, nibh", + {"Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit", + "Integer", "mattis", "nibh" + } + }, + } + + for _,c in ipairs(cases) do + local w = util.tokenize_text(c[1]) + assert_not_nil(w, "cannot tokenize " .. c[1]) + + for i,wrd in ipairs(w) do + logger.infox('%1:%2', i, wrd) + assert_equal(wrd, c[2][i]) + end + end + end) +end) \ No newline at end of file -- 2.39.5