diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2017-04-13 17:47:57 +0100 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2017-04-13 17:48:13 +0100 |
| commit | 23d9a026aa817ebce23cd28cb50be559e6f85896 (patch) | |
| tree | 571544e530935276fa938a1ee49dad599b66e9dc /src/libstat/tokenizers/osb.c | |
| parent | dbff4c2f3c72e8946f79bbcbc2e1e2262b3ce458 (diff) | |
| download | rspamd-23d9a026aa817ebce23cd28cb50be559e6f85896.tar.gz rspamd-23d9a026aa817ebce23cd28cb50be559e6f85896.zip | |
[Feature] Add unigramms support in bayes
Diffstat (limited to 'src/libstat/tokenizers/osb.c')
| -rw-r--r-- | src/libstat/tokenizers/osb.c | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/src/libstat/tokenizers/osb.c b/src/libstat/tokenizers/osb.c index f0e351bb5..54668e758 100644 --- a/src/libstat/tokenizers/osb.c +++ b/src/libstat/tokenizers/osb.c @@ -326,6 +326,18 @@ rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, } } + if (token_flags & RSPAMD_STAT_TOKEN_FLAG_UNIGRAM) { + new_tok = rspamd_mempool_alloc0 (pool, token_size); + new_tok->flags = token_flags; + new_tok->t1 = token; + new_tok->t2 = token; + new_tok->data = cur; + new_tok->window_idx = 0; + g_ptr_array_add (result, new_tok); + + continue; + } + #define ADD_TOKEN do {\ new_tok = rspamd_mempool_alloc0 (pool, token_size); \ new_tok->flags = token_flags; \ |