diff options
Diffstat (limited to 'src/tokenizers/tokenizers.h')
-rw-r--r-- | src/tokenizers/tokenizers.h | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/tokenizers/tokenizers.h b/src/tokenizers/tokenizers.h index c3453a945..10c8ae7aa 100644 --- a/src/tokenizers/tokenizers.h +++ b/src/tokenizers/tokenizers.h @@ -22,7 +22,7 @@ typedef struct token_node_s { /* Common tokenizer structure */ struct tokenizer { char *name; - GTree* (*tokenize_func)(struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input); + int (*tokenize_func)(struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input, GTree **cur); f_str_t* (*get_next_word)(f_str_t *buf, f_str_t *token); }; @@ -33,7 +33,7 @@ struct tokenizer* get_tokenizer (char *name); /* Get next word from specified f_str_t buf */ f_str_t *get_next_word (f_str_t *buf, f_str_t *token); /* OSB tokenize function */ -GTree* osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input); +int osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input, GTree **cur); /* Array of all defined tokenizers */ extern struct tokenizer tokenizers[]; |