diff options
Diffstat (limited to 'src/tokenizers/tokenizers.h')
| -rw-r--r-- | src/tokenizers/tokenizers.h | 24 |
1 files changed, 16 insertions, 8 deletions
diff --git a/src/tokenizers/tokenizers.h b/src/tokenizers/tokenizers.h index 207602dc8..883f38058 100644 --- a/src/tokenizers/tokenizers.h +++ b/src/tokenizers/tokenizers.h @@ -19,22 +19,30 @@ typedef struct token_node_s { /* Common tokenizer structure */ struct tokenizer { gchar *name; - gint (*tokenize_func)(struct tokenizer *tokenizer, rspamd_mempool_t *pool, f_str_t *input, - GTree **cur, gboolean save_token, gboolean is_utf, GList *exceptions); - gchar* (*get_next_word)(f_str_t *buf, f_str_t *token, GList **exceptions); + gint (*tokenize_func)(struct tokenizer *tokenizer, rspamd_mempool_t *pool, + f_str_t *input, + GTree **cur, gboolean save_token, gboolean is_utf, GList *exceptions); + gchar * (*get_next_word)(f_str_t *buf, f_str_t *token, GList **exceptions); }; /* Compare two token nodes */ int token_node_compare_func (gconstpointer a, gconstpointer b); /* Get tokenizer structure by name or return NULL if this name is not found */ -struct tokenizer* get_tokenizer (const char *name); +struct tokenizer * get_tokenizer (const char *name); /* Get next word from specified f_str_t buf */ -gchar* get_next_word (f_str_t *buf, f_str_t *token, GList **exceptions); +gchar * get_next_word (f_str_t *buf, f_str_t *token, GList **exceptions); /* OSB tokenize function */ -int osb_tokenize_text (struct tokenizer *tokenizer, rspamd_mempool_t *pool, f_str_t *input, - GTree **cur, gboolean save_token, gboolean is_utf, GList *exceptions); +int osb_tokenize_text (struct tokenizer *tokenizer, + rspamd_mempool_t *pool, + f_str_t *input, + GTree **cur, + gboolean save_token, + gboolean is_utf, + GList *exceptions); /* Common tokenizer for headers */ -int tokenize_headers (rspamd_mempool_t *pool, struct rspamd_task *task, GTree **cur); +int tokenize_headers (rspamd_mempool_t *pool, + struct rspamd_task *task, + GTree **cur); /* Make tokens for a subject */ void tokenize_subject (struct rspamd_task *task, GTree ** tree); |