summaryrefslogtreecommitdiffstats
path: root/src/tokenizers/tokenizers.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizers/tokenizers.h')
-rw-r--r--src/tokenizers/tokenizers.h4
1 files changed, 2 insertions, 2 deletions
diff --git a/src/tokenizers/tokenizers.h b/src/tokenizers/tokenizers.h
index c3453a945..10c8ae7aa 100644
--- a/src/tokenizers/tokenizers.h
+++ b/src/tokenizers/tokenizers.h
@@ -22,7 +22,7 @@ typedef struct token_node_s {
/* Common tokenizer structure */
struct tokenizer {
char *name;
- GTree* (*tokenize_func)(struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input);
+ int (*tokenize_func)(struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input, GTree **cur);
f_str_t* (*get_next_word)(f_str_t *buf, f_str_t *token);
};
@@ -33,7 +33,7 @@ struct tokenizer* get_tokenizer (char *name);
/* Get next word from specified f_str_t buf */
f_str_t *get_next_word (f_str_t *buf, f_str_t *token);
/* OSB tokenize function */
-GTree* osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input);
+int osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t *pool, f_str_t *input, GTree **cur);
/* Array of all defined tokenizers */
extern struct tokenizer tokenizers[];