diff options
| author | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2016-01-05 16:59:02 +0000 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@highsecure.ru> | 2016-01-05 16:59:02 +0000 |
| commit | a142fd150c47668215f8cf9f75374b8e8434b7d9 (patch) | |
| tree | 5cfa570a615e1f37ab57cee8f2b953094a7ad850 /src/libstat/tokenizers/tokenizers.h | |
| parent | 46c0c532f5bcc555cd106a61a5e659706290ac78 (diff) | |
| download | rspamd-a142fd150c47668215f8cf9f75374b8e8434b7d9.tar.gz rspamd-a142fd150c47668215f8cf9f75374b8e8434b7d9.zip | |
Fix tokenization
Diffstat (limited to 'src/libstat/tokenizers/tokenizers.h')
| -rw-r--r-- | src/libstat/tokenizers/tokenizers.h | 35 |
1 files changed, 10 insertions, 25 deletions
diff --git a/src/libstat/tokenizers/tokenizers.h b/src/libstat/tokenizers/tokenizers.h index f4c9a5ed3..70ff7560c 100644 --- a/src/libstat/tokenizers/tokenizers.h +++ b/src/libstat/tokenizers/tokenizers.h @@ -10,23 +10,19 @@ #define RSPAMD_DEFAULT_TOKENIZER "osb" struct rspamd_tokenizer_runtime; +struct rspamd_stat_ctx; /* Common tokenizer structure */ struct rspamd_stat_tokenizer { gchar *name; gpointer (*get_config) (rspamd_mempool_t *pool, struct rspamd_tokenizer_config *cf, gsize *len); - gboolean (*compatible_config) (struct rspamd_tokenizer_runtime *rt, - gpointer ptr, gsize len); - gboolean (*load_config) (rspamd_mempool_t *pool, - struct rspamd_tokenizer_runtime *rt, - gpointer ptr, gsize len); - gboolean (*is_compat) (struct rspamd_tokenizer_runtime *rt); - gint (*tokenize_func)(struct rspamd_tokenizer_runtime *rt, + gint (*tokenize_func)(struct rspamd_stat_ctx *ctx, rspamd_mempool_t *pool, GArray *words, gboolean is_utf, - const gchar *prefix); + const gchar *prefix, + GPtrArray *result); }; /* Compare two token nodes */ @@ -39,28 +35,17 @@ GArray * rspamd_tokenize_text (gchar *text, gsize len, gboolean is_utf, guint64 *hash); /* OSB tokenize function */ -gint rspamd_tokenizer_osb (struct rspamd_tokenizer_runtime *rt, - rspamd_mempool_t *pool, - GArray *input, - gboolean is_utf, - const gchar *prefix); +gint rspamd_tokenizer_osb (struct rspamd_stat_ctx *ctx, + rspamd_mempool_t *pool, + GArray *words, + gboolean is_utf, + const gchar *prefix, + GPtrArray *result); gpointer rspamd_tokenizer_osb_get_config (rspamd_mempool_t *pool, struct rspamd_tokenizer_config *cf, gsize *len); -gboolean -rspamd_tokenizer_osb_compatible_config (struct rspamd_tokenizer_runtime *rt, - gpointer ptr, gsize len); - -gboolean -rspamd_tokenizer_osb_load_config (rspamd_mempool_t *pool, - struct rspamd_tokenizer_runtime *rt, - gpointer ptr, gsize len); - -gboolean -rspamd_tokenizer_osb_is_compat (struct rspamd_tokenizer_runtime *rt); - #endif /* * vi:ts=4 |